validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py

@@ -2,27 +2,165 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-from functools import partial
-from typing import List
+from typing import Callable, Dict, List, Tuple, Union
 
 import matplotlib.pyplot as plt
 import pandas as pd
 import seaborn as sns
 from sklearn import metrics
 
-from validmind.vm_models import (
-    Figure,
-    ResultSummary,
-    ResultTable,
-    ResultTableMetadata,
-    ThresholdTest,
-    ThresholdTestResult,
-)
+from validmind.tests import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+DEFAULT_METRICS = {
+    "accuracy": metrics.accuracy_score,
+    "precision": metrics.precision_score,
+    "recall": metrics.recall_score,
+    "f1": metrics.f1_score,
+}
+DEFAULT_THRESHOLDS = {
+    "accuracy": 0.75,
+    "precision": 0.5,
+    "recall": 0.5,
+    "f1": 0.7,
+}
+
+
+def _compute_metrics(
+    results: dict,
+    metrics: Dict[str, Callable],
+    region: str,
+    df_region: pd.DataFrame,
+    target_column: str,
+    prediction_column: str,
+    feature_column: str,
+) -> None:
+    """
+    Computes and appends the default metrics for a given DataFrame slice to the results dictionary.
+    Args:
+        results (dict): A dictionary to which the computed metrics will be appended.
+        region (str): A string identifier for the DataFrame slice being evaluated.
+        df_region (pd.DataFrame): A pandas DataFrame slice containing the data to evaluate.
+        target_column (str): The name of the target column to use for computing the metrics.
+        prediction_column (str): The name of the prediction column to use for computing the metrics.
+    Returns:
+        None: The computed metrics are appended to the `results` dictionary in-place.
+    """
+    results["Slice"].append(str(region))
+    results["Shape"].append(df_region.shape[0])
+    results["Feature"].append(feature_column)
+
+    # Check if df_region is an empty dataframe and if so, append 0 to all metrics
+    if df_region.empty:
+        for metric in metrics.keys():
+            results[metric].append(0)
+        return
 
+    y_true = df_region[target_column].values
+    y_prediction = (
+        df_region[prediction_column].astype(df_region[target_column].dtypes).values
+    )
 
-@dataclass
-class WeakspotsDiagnosis(ThresholdTest):
+    for metric, metric_fn in metrics.items():
+        results[metric].append(metric_fn(y_true, y_prediction))
+
+
+def _plot_weak_spots(
+    results_1: dict, results_2: dict, feature_column: str, metric: str, threshold: float
+) -> Tuple[plt.Figure, pd.DataFrame]:
+    """
+    Plots the metric of the training and test datasets for each region in a given feature column,
+    and highlights regions where the score is below a specified threshold.
+    Args:
+        results_1 (list of dict): The results of the model on the training dataset, as a list of dictionaries.
+        results_2 (list of dict): The results of the model on the test dataset, as a list of dictionaries.
+        feature_column (str): The name of the feature column being analyzed.
+        metric (str): The name of the metric to plot.
+        threshold (float): The minimum accuracy threshold to be highlighted on the plot.
+    Returns:
+        fig (matplotlib.figure.Figure): The figure object containing the plot.
+        df (pandas.DataFrame): The concatenated dataframe containing the training and test results.
+    """
+    # Concat training and test datasets
+    results_1 = pd.DataFrame(results_1)
+    results_2 = pd.DataFrame(results_2)
+    dataset_type_column = "Dataset Type"
+    results_1[dataset_type_column] = "Training"
+    results_2[dataset_type_column] = "Test"
+    df = pd.concat([results_1, results_2])
+
+    # Create a bar plot using seaborn library
+    fig, ax = plt.subplots()
+    barplot = sns.barplot(
+        data=df,
+        x="Slice",
+        y=metric,
+        hue=dataset_type_column,
+        edgecolor="black",
+        ax=ax,
+    )
+    ax.tick_params(axis="x", rotation=90)
+    for p in ax.patches:
+        t = ax.annotate(
+            str("{:.2f}%".format(p.get_height())),
+            xy=(p.get_x() + 0.03, p.get_height() + 1),
+        )
+        t.set(color="black", size=14)
+
+    axhline = ax.axhline(
+        y=threshold,
+        color="red",
+        linestyle="--",
+        linewidth=3,
+        label=f"Threshold: {threshold}",
+    )
+    ax.set_ylabel(metric.capitalize(), weight="bold", fontsize=18)
+    ax.set_xlabel("Slice/Segments", weight="bold", fontsize=18)
+    ax.set_title(
+        f"Weak regions in feature column: {feature_column}",
+        weight="bold",
+        fontsize=20,
+        wrap=True,
+    )
+
+    # Get the legend handles and labels from the barplot
+    handles, labels = barplot.get_legend_handles_labels()
+
+    # Append the axhline handle and label
+    handles.append(axhline)
+    labels.append(axhline.get_label())
+
+    # Create a legend with both hue and axhline labels, the threshold line
+    # will show up twice so remove the first element
+    # barplot.legend(handles=handles[:-1], labels=labels, loc="upper right")
+    barplot.legend(
+        handles=handles[:-1],
+        labels=labels[:-1],
+        loc="upper center",
+        bbox_to_anchor=(0.5, 0.1),
+        ncol=len(handles) - 1,
+    )
+
+    plt.close()
+
+    return fig, df
+
+
+@tags(
+    "sklearn",
+    "binary_classification",
+    "multiclass_classification",
+    "model_diagnosis",
+    "visualization",
+)
+@tasks("classification", "text_classification")
+def WeakspotsDiagnosis(
+    datasets: List[VMDataset],
+    model: VMModel,
+    features_columns: Union[List[str], None] = None,
+    metrics: Union[Dict[str, Callable], None] = None,
+    thresholds: Union[Dict[str, float], None] = None,
+):
     """
     Identifies and visualizes weak spots in a machine learning model's performance across various sections of the
     feature space.
@@ -72,282 +210,88 @@ class WeakspotsDiagnosis(ThresholdTest):
     - Despite its usefulness in highlighting problematic regions, the test does not offer direct suggestions for model
     improvement.
     """
+    feature_columns = features_columns or datasets[0].feature_columns
+    if not all(col in datasets[0].feature_columns for col in feature_columns):
+        raise ValueError(
+            "Column(s) provided in features_columns do not exist in the dataset"
+        )
 
-    name = "weak_spots"
-    required_inputs = ["model", "datasets"]
-
-    default_params = {
-        "features_columns": None,
-        # Some default values that the user should override
-        "thresholds": {
-            "accuracy": 0.75,
-            "precision": 0.5,
-            "recall": 0.5,
-            "f1": 0.7,
-        },
-    }
-
-    tasks = ["classification", "text_classification"]
-    tags = [
-        "sklearn",
-        "binary_classification",
-        "multiclass_classification",
-        "model_diagnosis",
-        "visualization",
-    ]
+    metrics = metrics or DEFAULT_METRICS
+    metrics = {k.title(): v for k, v in metrics.items()}
 
-    # TODO: allow configuring
-    default_metrics = {
-        "accuracy": metrics.accuracy_score,
-        "precision": partial(metrics.precision_score, zero_division=0),
-        "recall": partial(metrics.recall_score, zero_division=0),
-        "f1": partial(metrics.f1_score, zero_division=0),
-    }
-
-    def run(self):
-        thresholds = self.params["thresholds"]
-
-        # Ensure there is a threshold for each metric
-        for metric in self.default_metrics.keys():
-            if metric not in thresholds:
-                raise ValueError(f"Threshold for metric {metric} is missing")
-
-        if self.params["features_columns"] is None:
-            features_list = self.inputs.datasets[0].feature_columns
-        else:
-            features_list = self.params["features_columns"]
-
-        if self.inputs.datasets[0].text_column in features_list:
-            raise ValueError(
-                "Skiping Weakspots Diagnosis test for the dataset with text column"
-            )
+    thresholds = thresholds or DEFAULT_THRESHOLDS
+    thresholds = {k.title(): v for k, v in thresholds.items()}
 
-        # Check if all elements from features_list are present in the feature columns
-        all_present = all(
-            elem in self.inputs.datasets[0].feature_columns for elem in features_list
-        )
-        if not all_present:
-            raise ValueError(
-                "The list of feature columns provided do not match with "
-                + "training dataset feature columns"
-            )
+    results_headers = ["Slice", "Shape", "Feature"]
+    results_headers.extend(metrics.keys())
 
-        target_column = self.inputs.datasets[0].target_column
-        prediction_column = f"{target_column}_pred"
-
-        train_df = self.inputs.datasets[0].df.copy()
-        train_class_pred = self.inputs.datasets[0].y_pred(self.inputs.model)
-        train_df[prediction_column] = train_class_pred
-
-        test_df = self.inputs.datasets[1].df.copy()
-        test_class_pred = self.inputs.datasets[1].y_pred(self.inputs.model)
-        test_df[prediction_column] = test_class_pred
-
-        test_results = []
-        test_figures = []
-        results_headers = ["slice", "shape", "feature"]
-        results_headers.extend(self.default_metrics.keys())
-        for feature in features_list:
-            bins = 10
-            if feature in self.inputs.datasets[0].feature_columns_categorical:
-                bins = len(train_df[feature].unique())
-            train_df["bin"] = pd.cut(train_df[feature], bins=bins)
-
-            results_train = {k: [] for k in results_headers}
-            results_test = {k: [] for k in results_headers}
-
-            for region, df_region in train_df.groupby("bin"):
-                self._compute_metrics(
-                    results_train,
-                    region,
-                    df_region,
-                    target_column,
-                    prediction_column,
-                    feature,
-                )
-                df_test_region = test_df[
-                    (test_df[feature] > region.left)
-                    & (test_df[feature] <= region.right)
-                ]
-                self._compute_metrics(
-                    results_test,
-                    region,
-                    df_test_region,
-                    target_column,
-                    prediction_column,
-                    feature,
-                )
-
-            # Make one plot per metric
-            for metric in self.default_metrics.keys():
-                fig, df = self._plot_weak_spots(
-                    results_train,
-                    results_test,
-                    feature,
-                    metric=metric,
-                    threshold=thresholds[metric],
-                )
-
-                test_figures.append(
-                    Figure(
-                        for_object=self,
-                        key=f"{self.name}:{metric}:{feature}",
-                        figure=fig,
-                        metadata={
-                            "metric": metric,
-                            "threshold": thresholds[metric],
-                            "feature": feature,
-                        },
-                    )
-                )
-
-            # For simplicity, test has failed if any of the metrics is below the threshold. We will
-            # rely on visual assessment for this test for now.
-            results_passed = df[df[list(thresholds.keys())].lt(thresholds).any(axis=1)]
-            passed = results_passed.empty
-
-            test_results.append(
-                ThresholdTestResult(
-                    test_name="accuracy",
-                    column=feature,
-                    passed=passed,
-                    values={"records": df.to_dict("records")},
-                )
-            )
-        return self.cache_results(
-            test_results,
-            passed=all([r.passed for r in test_results]),
-            figures=test_figures,
-        )
+    figures = []
+    passed = True
 
-    def summary(self, results: List[ThresholdTestResult], all_passed: bool):
-        results_table = [
-            record for result in results for record in result.values["records"]
-        ]
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=results_table,
-                    metadata=ResultTableMetadata(title="Weakspots Test"),
-                )
-            ]
-        )
+    df_1 = datasets[0]._df[
+        feature_columns
+        + [datasets[0].target_column, datasets[0].prediction_column(model)]
+    ]
+    df_2 = datasets[1]._df[
+        feature_columns
+        + [datasets[1].target_column, datasets[1].prediction_column(model)]
+    ]
 
-    def _compute_metrics(
-        self,
-        results: dict,
-        region: str,
-        df_region: pd.DataFrame,
-        target_column: str,
-        prediction_column: str,
-        feature_column: str,
-    ) -> None:
-        """
-        Computes and appends the default metrics for a given DataFrame slice to the results dictionary.
-        Args:
-            results (dict): A dictionary to which the computed metrics will be appended.
-            region (str): A string identifier for the DataFrame slice being evaluated.
-            df_region (pd.DataFrame): A pandas DataFrame slice containing the data to evaluate.
-            target_column (str): The name of the target column to use for computing the metrics.
-            prediction_column (str): The name of the prediction column to use for computing the metrics.
-        Returns:
-            None: The computed metrics are appended to the `results` dictionary in-place.
-        """
-        results["slice"].append(str(region))
-        results["shape"].append(df_region.shape[0])
-        results["feature"].append(feature_column)
-
-        # Check if df_region is an empty dataframe and if so, append 0 to all metrics
-        if df_region.empty:
-            for metric in self.default_metrics.keys():
-                results[metric].append(0)
-            return
-
-        y_true = df_region[target_column].values
-        y_prediction = (
-            df_region[prediction_column].astype(df_region[target_column].dtypes).values
-        )
+    for feature in feature_columns:
+        bins = 10
+        if feature in datasets[0].feature_columns_categorical:
+            bins = len(df_1[feature].unique())
+        df_1["bin"] = pd.cut(df_1[feature], bins=bins)
+
+        results_1 = {k: [] for k in results_headers}
+        results_2 = {k: [] for k in results_headers}
+
+        for region, df_region in df_1.groupby("bin"):
+            _compute_metrics(
+                results=results_1,
+                metrics=metrics,
+                region=region,
+                df_region=df_region,
+                target_column=datasets[0].target_column,
+                prediction_column=datasets[0].prediction_column(model),
+                feature_column=feature,
+            )
+            df_2_region = df_2[
+                (df_2[feature] > region.left) & (df_2[feature] <= region.right)
+            ]
+            _compute_metrics(
+                results=results_2,
+                metrics=metrics,
+                region=region,
+                df_region=df_2_region,
+                target_column=datasets[1].target_column,
+                prediction_column=datasets[1].prediction_column(model),
+                feature_column=feature,
+            )
 
-        for metric, metric_fn in self.default_metrics.items():
-            results[metric].append(metric_fn(y_true, y_prediction))
-
-    def _plot_weak_spots(
-        self, results_train, results_test, feature_column, metric, threshold
-    ):
-        """
-        Plots the metric of the training and test datasets for each region in a given feature column,
-        and highlights regions where the score is below a specified threshold.
-        Args:
-            results_train (list of dict): The results of the model on the training dataset, as a list of dictionaries.
-            results_test (list of dict): The results of the model on the test dataset, as a list of dictionaries.
-            feature_column (str): The name of the feature column being analyzed.
-            metric (str): The name of the metric to plot.
-            threshold (float): The minimum accuracy threshold to be highlighted on the plot.
-        Returns:
-            fig (matplotlib.figure.Figure): The figure object containing the plot.
-            df (pandas.DataFrame): The concatenated dataframe containing the training and test results.
-        """
-        # Concat training and test datasets
-        results_train = pd.DataFrame(results_train)
-        results_test = pd.DataFrame(results_test)
-        dataset_type_column = "Dataset Type"
-        results_train[dataset_type_column] = "Training"
-        results_test[dataset_type_column] = "Test"
-        df = pd.concat([results_train, results_test])
-
-        # Create a bar plot using seaborn library
-        fig, ax = plt.subplots()
-        barplot = sns.barplot(
-            data=df,
-            x="slice",
-            y=metric,
-            hue=dataset_type_column,
-            edgecolor="black",
-            ax=ax,
-        )
-        ax.tick_params(axis="x", rotation=90)
-        for p in ax.patches:
-            t = ax.annotate(
-                str("{:.2f}%".format(p.get_height())),
-                xy=(p.get_x() + 0.03, p.get_height() + 1),
+        for metric in metrics.keys():
+            fig, df = _plot_weak_spots(
+                results_1=results_1,
+                results_2=results_2,
+                feature_column=feature,
+                metric=metric,
+                threshold=thresholds[metric],
             )
-            t.set(color="black", size=14)
-
-        axhline = ax.axhline(
-            y=threshold,
-            color="red",
-            linestyle="--",
-            linewidth=3,
-            label=f"Threshold: {threshold}",
-        )
-        ax.set_ylabel(metric.capitalize(), weight="bold", fontsize=18)
-        ax.set_xlabel("Slice/Segments", weight="bold", fontsize=18)
-        ax.set_title(
-            f"Weak regions in feature column: {feature_column}",
-            weight="bold",
-            fontsize=20,
-            wrap=True,
-        )
 
-        # Get the legend handles and labels from the barplot
-        handles, labels = barplot.get_legend_handles_labels()
-
-        # Append the axhline handle and label
-        handles.append(axhline)
-        labels.append(axhline.get_label())
-
-        # Create a legend with both hue and axhline labels, the threshold line
-        # will show up twice so remove the first element
-        # barplot.legend(handles=handles[:-1], labels=labels, loc="upper right")
-        barplot.legend(
-            handles=handles[:-1],
-            labels=labels[:-1],
-            loc="upper center",
-            bbox_to_anchor=(0.5, 0.1),
-            ncol=len(handles) - 1,
-        )
+            figures.append(fig)
 
-        # Do this if you want to prevent the figure from being displayed
-        plt.close("all")
+        # For simplicity, test has failed if any of the metrics is below the threshold. We will
+        # rely on visual assessment for this test for now.
+        if not df[df[list(thresholds.keys())].lt(thresholds).any(axis=1)].empty:
+            passed = False
 
-        return fig, df
+    return (
+        pd.concat(
+            [
+                pd.DataFrame(results_1).assign(Dataset=datasets[0].input_id),
+                pd.DataFrame(results_2).assign(Dataset=datasets[1].input_id),
+            ]
+        ).sort_values(["Feature", "Dataset"]),
+        *figures,
+        passed,
+    )

validmind/tests/model_validation/statsmodels/AutoARIMA.py

@@ -5,13 +5,16 @@
 from statsmodels.tsa.arima.model import ARIMA
 from statsmodels.tsa.stattools import adfuller
 
+from validmind import tags, tasks
 from validmind.logging import get_logger
-from validmind.vm_models import Metric
+from validmind.vm_models import VMDataset, VMModel
 
 logger = get_logger(__name__)
 
 
-class AutoARIMA(Metric):
+@tags("time_series_data", "forecasting", "model_selection", "statsmodels")
+@tasks("regression")
+def AutoARIMA(model: VMModel, dataset: VMDataset):
     """
     Evaluates ARIMA models for time-series forecasting, ranking them using Bayesian and Akaike Information Criteria.
 
@@ -61,56 +64,48 @@ class AutoARIMA(Metric):
     - The test is only applicable to regression tasks involving time-series data, and may not work effectively for
     other types of machine learning tasks.
     """
-
-    name = "auto_arima"
-    required_inputs = ["dataset"]
-    tasks = ["regression"]
-    tags = ["time_series_data", "forecasting", "model_selection", "statsmodels"]
-
     max_p = 3
     max_d = 2
     max_q = 3
 
-    def run(self):
-        x_train = self.inputs.dataset.df
-
-        results = []
-
-        for col in x_train.columns:
-            series = x_train[col].dropna()
-
-            # Check for stationarity using the Augmented Dickey-Fuller test
-            adf_test = adfuller(series)
-            if adf_test[1] > 0.05:
-                logger.warning(
-                    f"Warning: {col} is not stationary. Results may be inaccurate."
-                )
-
-            arima_orders = []
-            bic_values = []
-            aic_values = []
-
-            for p in range(self.max_p + 1):
-                for d in range(self.max_d + 1):
-                    for q in range(self.max_q + 1):
-                        try:
-                            model = ARIMA(series, order=(p, d, q))
-                            model_fit = model.fit()
-
-                            arima_orders.append((p, d, q))
-                            bic_values.append(model_fit.bic)
-                            aic_values.append(model_fit.aic)
-                        except Exception as e:
-                            logger.error(
-                                f"Error fitting ARIMA({p}, {d}, {q}) model for {col}: {e}"
-                            )
-
-            result = {
+    df = dataset.x_df()
+
+    table = []
+
+    for col in df.columns:
+        series = df[col].dropna()
+
+        # Check for stationarity using the Augmented Dickey-Fuller test
+        adf_test = adfuller(series)
+        if adf_test[1] > 0.05:
+            logger.warning(f"{col} is not stationary. Results may be inaccurate.")
+
+        arima_orders = []
+        bic_values = []
+        aic_values = []
+
+        for p in range(max_p + 1):
+            for d in range(max_d + 1):
+                for q in range(max_q + 1):
+                    try:
+                        model = ARIMA(series, order=(p, d, q))
+                        model_fit = model.fit()
+
+                        arima_orders.append((p, d, q))
+                        bic_values.append(model_fit.bic)
+                        aic_values.append(model_fit.aic)
+                    except Exception as e:
+                        logger.error(
+                            f"Error fitting ARIMA({p}, {d}, {q}) model for {col}: {e}"
+                        )
+
+        table.append(
+            {
                 "Variable": col,
                 "ARIMA Orders": arima_orders,
                 "BIC": bic_values,
                 "AIC": aic_values,
             }
-            results.append(result)
+        )
 
-        return self.cache_results(results)
+    return table