validmind 2.5.8__py3-none-any.whl → 2.5.18__py3-none-any.whl

validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py

@@ -23,17 +23,19 @@ logger = get_logger(__name__)
 @dataclass
 class PopulationStabilityIndex(Metric):
     """
-    Evaluates the Population Stability Index (PSI) to quantify the stability of an ML model's predictions across
+    Assesses the Population Stability Index (PSI) to quantify the stability of an ML model's predictions across
     different datasets.
 
-    **Purpose:**
+    ### Purpose
+
     The Population Stability Index (PSI) serves as a quantitative assessment for evaluating the stability of a machine
     learning model's output distributions when comparing two different datasets. Typically, these would be a
     development and a validation dataset or two datasets collected at different periods. The PSI provides a measurable
     indication of any significant shift in the model's performance over time or noticeable changes in the
     characteristics of the population the model is making predictions for.
 
-    **Test Mechanism:**
+    ### Test Mechanism
+
     The implementation of the PSI in this script involves calculating the PSI for each feature between the training and
     test datasets. Data from both datasets is sorted and placed into either a predetermined number of bins or
     quantiles. The boundaries for these bins are initially determined based on the distribution of the training data.
@@ -42,14 +44,14 @@ class PopulationStabilityIndex(Metric):
     in the training and test datasets. The PSI, along with the proportions of data in each bin for both datasets, are
     displayed in a summary table, a grouped bar chart, and a scatter plot.
 
-    **Signs of High Risk:**
+    ### Signs of High Risk
 
     - A high PSI value is a clear indicator of high risk. Such a value suggests a significant shift in the model
     predictions or severe changes in the characteristics of the underlying population.
     - This ultimately suggests that the model may not be performing as well as expected and that it may be less
     reliable for making future predictions.
 
-    **Strengths:**
+    ### Strengths
 
     - The PSI provides a quantitative measure of the stability of a model over time or across different samples, making
     it an invaluable tool for evaluating changes in a model's performance.
@@ -58,7 +60,7 @@ class PopulationStabilityIndex(Metric):
     - The use of visual aids such as tables and charts further simplifies the comprehension and interpretation of the
     PSI.
 
-    **Limitations:**
+    ### Limitations
 
     - The PSI test does not account for the interdependence between features: features that are dependent on one
     another may show similar shifts in their distributions, which in turn may result in similar PSI values.

validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py

@@ -18,34 +18,41 @@ class PrecisionRecallCurve(Metric):
     """
     Evaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve.
 
-    **Purpose**: The Precision Recall Curve metric is intended to evaluate the trade-off between precision and recall
-    in classification models, particularly binary classification models. It assesses the model's capacity to produce
+    ### Purpose
+
+    The Precision Recall Curve metric is intended to evaluate the trade-off between precision and recall in
+    classification models, particularly binary classification models. It assesses the model's capacity to produce
     accurate results (high precision), as well as its ability to capture a majority of all positive instances (high
     recall).
 
-    **Test Mechanism**: The test extracts ground truth labels and prediction probabilities from the model's test
-    dataset. It applies the precision_recall_curve method from the sklearn metrics module to these extracted labels and
-    predictions, which computes a precision-recall pair for each possible threshold. This calculation results in an
-    array of precision and recall scores that can be plotted against each other to form the Precision-Recall Curve.
-    This curve is then visually represented by using Plotly's scatter plot.
+    ### Test Mechanism
+
+    The test extracts ground truth labels and prediction probabilities from the model's test dataset. It applies the
+    `precision_recall_curve` method from the sklearn metrics module to these extracted labels and predictions, which
+    computes a precision-recall pair for each possible threshold. This calculation results in an array of precision and
+    recall scores that can be plotted against each other to form the Precision-Recall Curve. This curve is then
+    visually represented by using Plotly's scatter plot.
+
+    ### Signs of High Risk
 
-    **Signs of High Risk**:
-    * A lower area under the Precision-Recall Curve signifies high risk.
-    * This corresponds to a model yielding a high amount of false positives (low precision) and/or false negatives (low
+    - A lower area under the Precision-Recall Curve signifies high risk.
+    - This corresponds to a model yielding a high amount of false positives (low precision) and/or false negatives (low
     recall).
-    * If the curve is closer to the bottom left of the plot, rather than being closer to the top right corner, it can
+    - If the curve is closer to the bottom left of the plot, rather than being closer to the top right corner, it can
     be a sign of high risk.
 
-    **Strengths**:
-    * This metric aptly represents the balance between precision (minimizing false positives) and recall (minimizing
+    ### Strengths
+
+    - This metric aptly represents the balance between precision (minimizing false positives) and recall (minimizing
     false negatives), which is especially critical in scenarios where both values are significant.
-    * Through the graphic representation, it enables an intuitive understanding of the model's performance across
+    - Through the graphic representation, it enables an intuitive understanding of the model's performance across
     different threshold levels.
 
-    **Limitations**:
-    * This metric is only applicable to binary classification models - it raises errors for multiclass classification
+    ### Limitations
+
+    - This metric is only applicable to binary classification models - it raises errors for multiclass classification
     models or Foundation models.
-    * It may not fully represent the overall accuracy of the model if the cost of false positives and false negatives
+    - It may not fully represent the overall accuracy of the model if the cost of false positives and false negatives
     are extremely different, or if the dataset is heavily imbalanced.
     """
 

validmind/tests/model_validation/sklearn/ROCCurve.py

@@ -19,7 +19,8 @@ class ROCCurve(Metric):
     Evaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic
     (ROC) curve and calculating the Area Under Curve (AUC) score.
 
-    **Purpose**:
+    ### Purpose
+
     The Receiver Operating Characteristic (ROC) curve is designed to evaluate the performance of binary classification
     models. This curve illustrates the balance between the True Positive Rate (TPR) and False Positive Rate (FPR)
     across various threshold levels. In combination with the Area Under the Curve (AUC), the ROC curve aims to measure
@@ -27,28 +28,32 @@ class ROCCurve(Metric):
     default vs non-default). Ideally, a higher AUC score signifies superior model performance in accurately
     distinguishing between the positive and negative classes.
 
-    **Test Mechanism**:
+    ### Test Mechanism
+
     First, this script selects the target model and datasets that require binary classification. It then calculates the
     predicted probabilities for the test set, and uses this data, along with the true outcomes, to generate and plot
-    the ROC curve. Additionally, it concludes a line signifying randomness (AUC of 0.5). The AUC score for the model's
+    the ROC curve. Additionally, it includes a line signifying randomness (AUC of 0.5). The AUC score for the model's
     ROC curve is also computed, presenting a numerical estimation of the model's performance. If any Infinite values
     are detected in the ROC threshold, these are effectively eliminated. The resulting ROC curve, AUC score, and
     thresholds are consequently saved for future reference.
 
-    **Signs of High Risk**:
+    ### Signs of High Risk
+
     - A high risk is potentially linked to the model's performance if the AUC score drops below or nears 0.5.
     - Another warning sign would be the ROC curve lying closer to the line of randomness, indicating no discriminative
     ability.
     - For the model to be deemed competent at its classification tasks, it is crucial that the AUC score is
     significantly above 0.5.
 
-    **Strengths**:
-    - This ROC Curve offers an inclusive visual depiction of a model's discriminative power throughout all conceivable
+    ### Strengths
+
+    - The ROC Curve offers an inclusive visual depiction of a model's discriminative power throughout all conceivable
     classification thresholds, unlike other metrics that solely disclose model performance at one fixed threshold.
     - Despite the proportions of the dataset, the AUC Score, which represents the entire ROC curve as a single data
     point, continues to be consistent, proving to be the ideal choice for such situations.
 
-    **Limitations**:
+    ### Limitations
+
     - The primary limitation is that this test is exclusively structured for binary classification tasks, thus limiting
     its application towards other model types.
     - Furthermore, its performance might be subpar with models that output probabilities highly skewed towards 0 or 1.

validmind/tests/model_validation/sklearn/RegressionErrors.py

@@ -2,141 +2,85 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import numpy as np
+import pandas as pd
 from sklearn import metrics
 
-from validmind.vm_models import Metric, ResultSummary, ResultTable
+from validmind import tags, tasks
 
 
-@dataclass
-class RegressionErrors(Metric):
+@tags("sklearn", "model_performance")
+@tasks("regression", "classification")
+def RegressionErrors(model, dataset):
     """
-    **Purpose**: This metric is used to measure the performance of a regression model. It gauges the model's accuracy
-    by computing several error metrics such as Mean Absolute Error (MAE), Mean Squared Error (MSE), Root Mean Squared
-    Error (RMSE), Mean Absolute Percentage Error (MAPE), and Mean Bias Deviation (MBD) on both the training and testing
-    dataset.
-
-    **Test Mechanism**: The test computes each of the aforementioned metrics. MAE calculates the average of the
-    absolute difference between the true value and the predicted value. MSE squares the difference before averaging it.
-    RMSE then takes the square root of the MSE. MAPE evaluates the average of the absolute difference between true and
-    predicted values divided by the true value, expressed as a percentage. Lastly, MBD is a measure of average bias in
-    the prediction. The results are compared between the training dataset and the testing dataset.
-
-    **Signs of High Risk**: High values for any of the metrics, or particularly different metric outcomes for the
-    training set versus the test set, are signs of high risk. Specifically, high MAE, MSE, RMSE, or MAPE values could
-    indicate poor model performance and overfitting. If MBD is significantly different from zero, it could signify that
-    the model's predictions are systematically biased.
-
-    **Strengths**: These metrics collectively provide a comprehensive view of model performance and error distribution.
-    Individually, MAE provides a linear score that could be more interpretable, while MSE gives more weight to larger
-    errors. RMSE is useful because it is in the same unit as the target variable. MAPE expresses error as a percentage,
-    making it a good measure of prediction accuracy. MBD helps to detect systematic bias in predictions.
-
-    **Limitations**: Each of these metrics has its own limitations. MAE and MSE are sensitive to outliers. While RMSE
-    is good for giving high weight to larger errors, it might too heavily penalize these errors. MAPE might be biased
-    if actual values are near zero, and MBD would not work well if the difference between predictions and actual values
-    changes with the magnitude of the actual values. Overall, these metrics will not capture all model performance
-    nuances, and they should be used with contextual understanding of the problem at hand.
+    Assesses the performance and error distribution of a regression model using various error metrics.
+
+    ### Purpose
+
+    The purpose of the Regression Errors test is to measure the performance of a regression model by calculating
+    several error metrics. This evaluation helps determine the model's accuracy and potential issues like overfitting
+    or bias by analyzing differences in error metrics between the training and testing datasets.
+
+    ### Test Mechanism
+
+    The test computes the following error metrics:
+    - **Mean Absolute Error (MAE)**: Average of the absolute differences between true values and predicted values.
+    - **Mean Squared Error (MSE)**: Average of the squared differences between true values and predicted values.
+    - **Root Mean Squared Error (RMSE)**: Square root of the mean squared error.
+    - **Mean Absolute Percentage Error (MAPE)**: Average of the absolute differences between true values and predicted
+    values, divided by the true values, and expressed as a percentage.
+    - **Mean Bias Deviation (MBD)**: Average bias between true values and predicted values.
+
+    These metrics are calculated separately for the training and testing datasets and compared to identify
+    discrepancies.
+
+    ### Signs of High Risk
+
+    - High values for MAE, MSE, RMSE, or MAPE indicating poor model performance.
+    - Large differences in error metrics between the training and testing datasets, suggesting overfitting.
+    - Significant deviation of MBD from zero, indicating systematic bias in model predictions.
+
+    ### Strengths
+
+    - Provides a comprehensive overview of model performance through multiple error metrics.
+    - Individual metrics offer specific insights, e.g., MAE for interpretability, MSE for emphasizing larger errors.
+    - RMSE is useful for being in the same unit as the target variable.
+    - MAPE allows the error to be expressed as a percentage.
+    - MBD detects systematic bias in model predictions.
+
+    ### Limitations
+
+    - MAE and MSE are sensitive to outliers.
+    - RMSE heavily penalizes larger errors, which might not always be desirable.
+    - MAPE can be misleading when actual values are near zero.
+    - MBD may not be suitable if bias varies with the magnitude of actual values.
+    - These metrics may not capture all nuances of model performance and should be interpreted with domain-specific
+    context.
     """
 
-    name = "regression_errors"
-    required_inputs = ["model", "datasets"]
-    tasks = ["regression"]
-    tags = [
-        "sklearn",
-        "model_performance",
-    ]
-
-    def summary(self, raw_results):
-        """
-        Returns a summarized representation of the dataset split information
-        """
-        table_records = []
-        for result in raw_results:
-            for key, _ in result.items():
-                table_records.append(
-                    {
-                        "Metric": key,
-                        "TRAIN": result[key]["train"],
-                        "TEST": result[key]["test"],
-                    }
-                )
-
-        return ResultSummary(results=[ResultTable(data=table_records)])
-
-    def regression_errors(
-        self, y_true_train, class_pred_train, y_true_test, class_pred_test
-    ):
-        mae_train = metrics.mean_absolute_error(y_true_train, class_pred_train)
-        mae_test = metrics.mean_absolute_error(y_true_test, class_pred_test)
-
-        results = []
-        results.append(
-            {
-                "Mean Absolute Error (MAE)": {
-                    "train": mae_train,
-                    "test": mae_test,
-                }
-            }
-        )
-
-        mse_train = metrics.mean_squared_error(y_true_train, class_pred_train)
-        mse_test = metrics.mean_squared_error(y_true_test, class_pred_test)
-        results.append(
-            {
-                "Mean Squared Error (MSE)": {
-                    "train": mse_train,
-                    "test": mse_test,
-                }
-            }
-        )
-        results.append(
-            {
-                "Root Mean Squared Error (RMSE)": {
-                    "train": np.sqrt(mse_train),
-                    "test": np.sqrt(mse_test),
-                }
-            }
-        )
-
-        mape_train = (
-            np.mean(np.abs((y_true_train - class_pred_train) / y_true_train)) * 100
-        )
-        mape_test = np.mean(np.abs((y_true_test - class_pred_test) / y_true_test)) * 100
-        results.append(
-            {
-                "Mean Absolute Percentage Error (MAPE)": {
-                    "train": mape_train,
-                    "test": mape_test,
-                }
-            }
-        )
-
-        mbd_train = np.mean(class_pred_train - y_true_train)
-        mbd_test = np.mean(class_pred_test - y_true_test)
-        results.append(
-            {
-                "Mean Bias Deviation (MBD)": {
-                    "train": mbd_train,
-                    "test": mbd_test,
-                }
-            }
-        )
-        return results
-
-    def run(self):
-        y_train_true = self.inputs.datasets[0].y
-        y_train_pred = self.inputs.datasets[0].y_pred(self.inputs.model)
-        y_train_true = y_train_true.astype(y_train_pred.dtype)
-
-        y_test_true = self.inputs.datasets[1].y
-        y_test_pred = self.inputs.datasets[1].y_pred(self.inputs.model)
-        y_test_true = y_test_true.astype(y_test_pred.dtype)
-
-        results = self.regression_errors(
-            y_train_true, y_train_pred, y_test_true, y_test_pred
-        )
-
-        return self.cache_results(metric_value=results)
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+    y_true = y_true.astype(y_pred.dtype)
+
+    return _regression_errors(y_true, y_pred)
+
+
+def _regression_errors(y_true, y_pred):
+    mae_train = metrics.mean_absolute_error(y_true, y_pred)
+    mse_train = metrics.mean_squared_error(y_true, y_pred)
+    rmse_train = np.sqrt(mse_train)
+    mape_train = np.mean(np.abs((y_true - y_pred) / y_true)) * 100
+    mbd_train = np.mean(y_pred - y_true)
+
+    # Create dataframe with one row and each error metric as a column
+    results_df = pd.DataFrame(
+        {
+            "Mean Absolute Error (MAE)": [mae_train],
+            "Mean Squared Error (MSE)": [mse_train],
+            "Root Mean Squared Error (RMSE)": [rmse_train],
+            "Mean Absolute Percentage Error (MAPE)": [mape_train],
+            "Mean Bias Deviation (MBD)": [mbd_train],
+        }
+    )
+
+    return results_df

validmind/tests/model_validation/sklearn/RegressionErrorsComparison.py

@@ -16,25 +16,40 @@ logger = get_logger(__name__)
 @tasks("regression", "time_series_forecasting")
 def RegressionErrorsComparison(datasets, models):
     """
-    Compare regression error metrics for each model and generate a summary table
-    with the results.
+    Assesses multiple regression error metrics to compare model performance across different datasets, emphasizing
+    systematic overestimation or underestimation and large percentage errors.
 
-    **Purpose**: The purpose of this function is to compare the regression errors for different models applied to various datasets.
+    ### Purpose
 
-    **Test Mechanism**: The function iterates through each dataset-model pair, calculates various error metrics (MAE, MSE, MAPE, MBD), and generates a summary table with these results.
+    The purpose of this test is to compare regression errors for different models applied to various datasets. It aims
+    to examine model performance using multiple error metrics, thereby identifying areas where models may be
+    underperforming or exhibiting bias.
 
-    **Signs of High Risk**:
-    - High Mean Absolute Error (MAE) or Mean Squared Error (MSE) indicates poor model performance.
-    - High Mean Absolute Percentage Error (MAPE) suggests large percentage errors, especially problematic if the true values are small.
-    - Mean Bias Deviation (MBD) significantly different from zero indicates systematic overestimation or underestimation by the model.
+    ### Test Mechanism
+
+    The function iterates through each dataset-model pair and calculates various error metrics, including Mean Absolute
+    Error (MAE), Mean Squared Error (MSE), Mean Absolute Percentage Error (MAPE), and Mean Bias Deviation (MBD). The
+    results are summarized in a table, which provides a comprehensive view of each model's performance on the datasets.
+
+    ### Signs of High Risk
+
+    - High Mean Absolute Error (MAE) or Mean Squared Error (MSE), indicating poor model performance.
+    - High Mean Absolute Percentage Error (MAPE), suggesting large percentage errors, especially problematic if the
+    true values are small.
+    - Mean Bias Deviation (MBD) significantly different from zero, indicating systematic overestimation or
+    underestimation by the model.
+
+    ### Strengths
 
-    **Strengths**:
     - Provides multiple error metrics to assess model performance from different perspectives.
     - Includes a check to avoid division by zero when calculating MAPE.
 
-    **Limitations**:
-    - Assumes that the dataset is provided as a DataFrameDataset object with `y`, `y_pred`, and `feature_columns` attributes.
-    - The function relies on the `logger` from `validmind.logging` to warn about zero values in `y_true`, which should be correctly implemented and imported.
+    ### Limitations
+
+    - Assumes that the dataset is provided as a DataFrameDataset object with `y`, `y_pred`, and `feature_columns`
+    attributes.
+    - Relies on the `logger` from `validmind.logging` to warn about zero values in `y_true`, which should be correctly
+    implemented and imported.
     - Requires that `dataset.y_pred(model)` returns the predicted values for the model.
     """
     results_list = []

validmind/tests/model_validation/sklearn/{RegressionModelsPerformanceComparison.py → RegressionPerformance.py}

@@ -16,25 +16,27 @@ logger = get_logger(__name__)
 
 
 @dataclass
-class RegressionModelsPerformanceComparison(Metric):
+class RegressionPerformance(Metric):
     """
     Compares and evaluates the performance of multiple regression models using five different metrics: MAE, MSE, RMSE,
     MAPE, and MBD.
 
-    **1. Purpose:**
+    ### Purpose
+
     The Regression Models Performance Comparison metric is used to measure and compare the performance of regression
     models. It calculates multiple evaluation metrics, including Mean Absolute Error (MAE), Mean Squared Error (MSE),
     Root Mean Squared Error (RMSE), Mean Absolute Percentage Error (MAPE), and Mean Bias Deviation (MBD), thereby
     enabling a comprehensive view of model performance.
 
-    **2. Test Mechanism:**
+    ### Test Mechanism
+
     The test starts by sourcing the true and predicted values from the models. It then computes the MAE, MSE, RMSE,
     MAPE, and MBD. These calculations encapsulate both the direction and the magnitude of error in predictions, thereby
     providing a multi-faceted view of model accuracy. It captures these results in a dictionary and compares the
     performance of all models using these metrics. The results are then appended to a table for presenting a
     comparative summary.
 
-    **3. Signs of High Risk:**
+    ### Signs of High Risk
 
     - High values of MAE, MSE, RMSE, and MAPE, which indicate a high error rate and imply a larger departure of the
     model's predictions from the true values.
@@ -42,13 +44,13 @@ class RegressionModelsPerformanceComparison(Metric):
     - If the test returns an error citing that no models were provided for comparison, it implies a risk in the
     evaluation process itself.
 
-    **4. Strengths:**
+    ### Strengths
 
     - The metric evaluates models on five different metrics offering a comprehensive analysis of model performance.
     - It compares multiple models simultaneously, aiding in the selection of the best-performing models.
     - It is designed to handle regression tasks and can be seamlessly integrated with libraries like sklearn.
 
-    **5. Limitations:**
+    ### Limitations
 
     - The metric only evaluates regression models and does not evaluate classification models.
     - The test assumes that the models have been trained and tested appropriately prior to evaluation. It does not
@@ -58,8 +60,8 @@ class RegressionModelsPerformanceComparison(Metric):
     - The test could exhibit performance limitations if a large number of models is input for comparison.
     """
 
-    name = "models_performance_comparison"
-    required_inputs = ["dataset", "models"]
+    name = "regression_performance"
+    required_inputs = ["dataset", "model"]
 
     tasks = ["regression"]
     tags = [
@@ -96,7 +98,7 @@ class RegressionModelsPerformanceComparison(Metric):
         This summary varies depending if we're evaluating a binary or multi-class model
         """
         results = []
-        metrics = metric_value["model_0"].keys()
+        metrics = metric_value[self.inputs.model.input_id].keys()
         error_table = []
         for metric_name in metrics:
             errors_dict = {}
@@ -119,20 +121,16 @@ class RegressionModelsPerformanceComparison(Metric):
 
     def run(self):
         # Check models list is not empty
-        if not self.inputs.models:
+        if not self.inputs.model:
             raise SkipTestError(
-                "List of models must be provided as a `models` parameter to compare performance"
+                "Model must be provided as a `models` parameter to compare performance"
             )
-
-        all_models = self.inputs.models
-
         results = {}
 
-        for idx, model in enumerate(all_models):
-            result = self.regression_errors(
-                y_true_test=self.inputs.dataset.y,
-                y_pred_test=self.inputs.dataset.y_pred(model),
-            )
-            results["model_" + str(idx)] = result
+        result = self.regression_errors(
+            y_true_test=self.inputs.dataset.y,
+            y_pred_test=self.inputs.dataset.y_pred(self.inputs.model),
+        )
+        results[self.inputs.model.input_id] = result
 
         return self.cache_results(results)