validmind 2.5.1__py3-none-any.whl → 2.5.6__py3-none-any.whl

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.5.1"
+__version__ = "2.5.6"

validmind/client.py

@@ -240,6 +240,11 @@ def init_model(
         vm_model = class_obj(
             pipeline=model,
             input_id=input_id,
+            attributes=(
+                ModelAttributes.from_dict(attributes)
+                if attributes
+                else ModelAttributes()
+            ),
         )
         # TODO: Add metadata for pipeline model
         metadata = get_model_info(vm_model)
@@ -248,6 +253,7 @@ def init_model(
             input_id=input_id,
             model=model,  # Trained model instance
             predict_fn=predict_fn,
+            attributes=ModelAttributes.from_dict(attributes) if attributes else None,
             **kwargs,
         )
         metadata = get_model_info(vm_model)

validmind/tests/__types__.py

@@ -56,7 +56,6 @@ TestID = Literal[
     "validmind.model_validation.ragas.AnswerSimilarity",
     "validmind.model_validation.ragas.AnswerCorrectness",
     "validmind.model_validation.ragas.ContextRecall",
-    "validmind.model_validation.ragas.ContextRelevancy",
     "validmind.model_validation.ragas.ContextPrecision",
     "validmind.model_validation.ragas.AnswerRelevance",
     "validmind.model_validation.sklearn.RegressionModelsPerformanceComparison",

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -105,7 +105,7 @@ def AnswerCorrectness(
         "ground_truth": ground_truth_column,
     }
 
-    df = get_renamed_columns(dataset.df, required_columns)
+    df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
         Dataset.from_pandas(df), metrics=[answer_correctness], **get_ragas_config()

validmind/tests/model_validation/ragas/AnswerRelevance.py

@@ -109,7 +109,7 @@ def AnswerRelevance(
         "contexts": contexts_column,
     }
 
-    df = get_renamed_columns(dataset.df, required_columns)
+    df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
         Dataset.from_pandas(df), metrics=[answer_relevancy], **get_ragas_config()

validmind/tests/model_validation/ragas/AnswerSimilarity.py

@@ -94,7 +94,7 @@ def AnswerSimilarity(
         "ground_truth": ground_truth_column,
     }
 
-    df = get_renamed_columns(dataset.df, required_columns)
+    df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
         Dataset.from_pandas(df), metrics=[answer_similarity], **get_ragas_config()

validmind/tests/model_validation/ragas/AspectCritique.py

@@ -132,7 +132,7 @@ def AspectCritique(
         "contexts": contexts_column,
     }
 
-    df = get_renamed_columns(dataset.df, required_columns)
+    df = get_renamed_columns(dataset._df, required_columns)
 
     built_in_aspects = [aspect_map[aspect] for aspect in aspects]
     custom_aspects = (

validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -100,7 +100,7 @@ def ContextEntityRecall(
         "contexts": contexts_column,
     }
 
-    df = get_renamed_columns(dataset.df, required_columns)
+    df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
         Dataset.from_pandas(df), metrics=[context_entity_recall], **get_ragas_config()

validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -96,7 +96,7 @@ def ContextPrecision(
         "ground_truth": ground_truth_column,
     }
 
-    df = get_renamed_columns(dataset.df, required_columns)
+    df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
         Dataset.from_pandas(df), metrics=[context_precision], **get_ragas_config()

validmind/tests/model_validation/ragas/ContextRecall.py

@@ -96,7 +96,7 @@ def ContextRecall(
         "ground_truth": ground_truth_column,
     }
 
-    df = get_renamed_columns(dataset.df, required_columns)
+    df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
         Dataset.from_pandas(df), metrics=[context_recall], **get_ragas_config()

validmind/tests/model_validation/ragas/Faithfulness.py

@@ -94,7 +94,7 @@ def Faithfulness(
         "contexts": contexts_column,
     }
 
-    df = get_renamed_columns(dataset.df, required_columns)
+    df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
         Dataset.from_pandas(df), metrics=[faithfulness], **get_ragas_config()

validmind/tests/model_validation/sklearn/OverfitDiagnosis.py

@@ -25,51 +25,48 @@ from validmind.vm_models import (
 
 logger = get_logger(__name__)
 
+# TODO: A couple of improvements here could be to:
+# 1. Allow the test to use multiple metrics at once
+# 2. Allow custom functions for computing performance
+
 DEFAULT_THRESHOLD = 0.04
+DEFAULT_CLASSIFICATION_METRIC = "auc"
+DEFAULT_REGRESSION_METRIC = "mse"
 PERFORMANCE_METRICS = {
     "accuracy": {
         "function": metrics.accuracy_score,
-        "is_classification": True,
         "is_lower_better": False,
     },
     "auc": {
         "function": metrics.roc_auc_score,
-        "is_classification": True,
         "is_lower_better": False,
     },
     "f1": {
         "function": metrics.f1_score,
-        "is_classification": True,
         "is_lower_better": False,
     },
     "precision": {
         "function": metrics.precision_score,
-        "is_classification": True,
         "is_lower_better": False,
     },
     "recall": {
         "function": metrics.recall_score,
-        "is_classification": True,
         "is_lower_better": False,
     },
     "mse": {
         "function": metrics.mean_squared_error,
-        "is_classification": False,
         "is_lower_better": True,
     },
     "mae": {
         "function": metrics.mean_absolute_error,
-        "is_classification": False,
         "is_lower_better": True,
     },
     "r2": {
         "function": metrics.r2_score,
-        "is_classification": False,
         "is_lower_better": False,
     },
     "mape": {
         "function": metrics.mean_absolute_percentage_error,
-        "is_classification": False,
         "is_lower_better": True,
     },
 }
@@ -123,20 +120,13 @@ def _compute_metrics(
     if is_classification and metric == "auc":
         # if only one class is present in the data, return 0
         if len(np.unique(y_true)) == 1:
-            results[metric].append(0)
-            return
-
-        score = metric_func(y_true, df_region[prob_column].values)
-
-    # All other classification metrics
-    elif is_classification:
-        score = metric_func(y_true, df_region[pred_column].values)
+            return results[metric].append(0)
 
-    # Regression metrics
-    else:
-        score = metric_func(y_true, df_region[pred_column].values)
+        return results[metric].append(
+            metric_func(y_true, df_region[prob_column].values)
+        )
 
-    results[metric].append(score)
+    return results[metric].append(metric_func(y_true, df_region[pred_column].values))
 
 
 def _plot_overfit_regions(
@@ -219,8 +209,12 @@ def overfit_diagnosis(  # noqa: C901
     is_classification = bool(datasets[0].probability_column(model))
 
     # Set default metric if not provided
-    if metric is None:
-        metric = "auc" if is_classification else "mse"
+    if not metric:
+        metric = (
+            DEFAULT_CLASSIFICATION_METRIC
+            if is_classification
+            else DEFAULT_REGRESSION_METRIC
+        )
         logger.info(
             f"Using default {'classification' if is_classification else 'regression'} metric: {metric}"
         )
@@ -228,19 +222,6 @@ def overfit_diagnosis(  # noqa: C901
     if id(cut_off_threshold) == id(DEFAULT_THRESHOLD):
         logger.info("Using default cut-off threshold of 0.04")
 
-    metric = metric.lower()
-    try:
-        _metric = PERFORMANCE_METRICS[metric.lower()]
-    except KeyError:
-        raise ValueError(
-            f"Invalid metric. Choose from: {', '.join(PERFORMANCE_METRICS.keys())}"
-        )
-
-    if is_classification and not _metric["is_classification"]:
-        raise ValueError(f"Cannot use regression metric ({metric}) for classification.")
-    elif not is_classification and _metric["is_classification"]:
-        raise ValueError(f"Cannot use classification metric ({metric}) for regression.")
-
     train_df = datasets[0].df
     test_df = datasets[1].df
 

validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py

@@ -2,6 +2,7 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from collections import defaultdict
 from dataclasses import dataclass
 from operator import add
 from typing import List, Tuple
@@ -13,6 +14,7 @@ import seaborn as sns
 from sklearn import metrics
 
 from validmind.errors import MissingOrInvalidModelPredictFnError
+from validmind.logging import get_logger
 from validmind.vm_models import (
     Figure,
     ResultSummary,
@@ -20,291 +22,348 @@ from validmind.vm_models import (
     ResultTableMetadata,
     ThresholdTest,
     ThresholdTestResult,
+    VMDataset,
+    VMModel,
 )
 
-
-# TODO: make this support regression and classification as well as more performance metrics
-@dataclass
-class RobustnessDiagnosis(ThresholdTest):
+logger = get_logger(__name__)
+
+DEFAULT_DECAY_THRESHOLD = 0.05
+DEFAULT_STD_DEV_LIST = [0.1, 0.2, 0.3, 0.4, 0.5]
+DEFAULT_CLASSIFICATION_METRIC = "auc"
+DEFAULT_REGRESSION_METRIC = "mse"
+PERFORMANCE_METRICS = {
+    "accuracy": {
+        "function": metrics.accuracy_score,
+        "is_lower_better": False,
+    },
+    "auc": {
+        "function": metrics.roc_auc_score,
+        "is_lower_better": False,
+    },
+    "f1": {
+        "function": metrics.f1_score,
+        "is_lower_better": False,
+    },
+    "precision": {
+        "function": metrics.precision_score,
+        "is_lower_better": False,
+    },
+    "recall": {
+        "function": metrics.recall_score,
+        "is_lower_better": False,
+    },
+    "mse": {
+        "function": metrics.mean_squared_error,
+        "is_lower_better": True,
+    },
+    "mae": {
+        "function": metrics.mean_absolute_error,
+        "is_lower_better": True,
+    },
+    "r2": {
+        "function": metrics.r2_score,
+        "is_lower_better": False,
+    },
+    "mape": {
+        "function": metrics.mean_absolute_percentage_error,
+        "is_lower_better": True,
+    },
+}
+
+
+def _add_noise_std_dev(
+    values: List[float], x_std_dev: float
+) -> Tuple[List[float], float]:
     """
-    Evaluates the robustness of a machine learning model by injecting Gaussian noise to input data and measuring
-    performance.
-
-    **Purpose**:
-
-    The purpose of this test code is to evaluate the robustness of a machine learning model. Robustness refers to a
-    model's ability to maintain a high level of performance in the face of perturbations or changes—particularly
-    noise—added to its input data. This test is designed to help gauge how well the model can handle potential
-    real-world scenarios where the input data might be incomplete or corrupted.
-
-    **Test Mechanism**:
-
-    This test is conducted by adding Gaussian noise, proportional to a particular standard deviation scale, to numeric
-    input features of both the training and testing datasets. The model performance in the face of these perturbed
-    features is then evaluated using the ROC_AUC score. This process is iterated over a range of scale
-    factors. The resulting auc trend against the amount of noise introduced is illustrated with a line chart. A
-    predetermined threshold determines what level of auc decay due to perturbation is considered acceptable.
-
-    **Signs of High Risk**:
-    - Substantial decreases in auc when noise is introduced to feature inputs.
-    - The decay in auc surpasses the configured threshold, indicating that the model is not robust against input
-    noise.
-    - Instances where one or more elements provided in the features list don't match with the training dataset's
-    numerical feature columns.
-
-    **Strengths**:
-    - Provides an empirical measure of the model's performance in tackling noise or data perturbations, revealing
-    insights into the model's stability.
-    - Offers flexibility with the ability to choose specific features to perturb and control the level of noise applied.
-    - Detailed results visualization helps in interpreting the outcome of robustness testing.
-
-    **Limitations**:
-    - The default threshold for auc decay is set to 0.05, which is unlikely to be optimal for most use cases and
-    should be adjusted based on domain expertise to suit the needs of the specific model.
-    - Only numerical features are perturbed, leaving out non-numerical features, which can lead to an incomplete
-    analysis of robustness.
-    - The test is contingent on the assumption that the added Gaussian noise sufficiently represents potential data
-    corruption or incompleteness in real-world scenarios.
+    Adds Gaussian noise to a list of values.
+    Args:
+        values (list[float]): A list of numerical values to which noise is added.
+        x_std_dev (float): A scaling factor for the standard deviation of the noise.
+    Returns:
+        tuple[list[float], float]: A tuple containing:
+            - A list of noisy values, where each value is the sum of the corresponding value
+            in the input list and a randomly generated value sampled from a Gaussian distribution
+            with mean 0 and standard deviation x_std_dev times the standard deviation of the input list.
+            - The standard deviation of the input list of values.
     """
+    std_dev = np.std(values)
+    noise_list = np.random.normal(0, x_std_dev * std_dev, size=len(values))
+    noisy_values = list(map(add, noise_list, values))
 
-    name = "robustness"
-    required_inputs = ["model", "datasets"]
-    default_params = {
-        "features_columns": None,
-        "scaling_factor_std_dev_list": [0.0, 0.1, 0.2, 0.3, 0.4, 0.5],
-        "auc_decay_threshold": 0.05,
-    }
-    tasks = ["classification"]
-    tags = [
-        "sklearn",
-        "binary_classification",
-        "multiclass_classification",
-        "model_diagnosis",
-        "visualization",
-    ]
+    return noisy_values
 
-    def run(self):
-        # Validate X std deviation parameter
-        if "scaling_factor_std_dev_list" not in self.params:
-            raise ValueError("scaling_factor_std_dev_list must be provided in params")
-        x_std_dev_list = self.params["scaling_factor_std_dev_list"]
 
-        if self.params["auc_decay_threshold"] is None:
-            raise ValueError("auc_decay_threshold must be provided in params")
-        auc_threshold = self.params["auc_decay_threshold"]
+def _compute_metric(
+    dataset: VMDataset, model: VMModel, X: pd.DataFrame, metric: str
+) -> float:
+    if metric not in PERFORMANCE_METRICS:
+        raise ValueError(
+            f"Invalid metric: {metric}, expected one of {PERFORMANCE_METRICS.keys()}"
+        )
 
-        if self.inputs.model is None:
-            raise ValueError("model must of provided to run this test")
+    if metric == "auc":
+        try:
+            y_proba = model.predict_proba(X)
+        except MissingOrInvalidModelPredictFnError:
+            y_proba = model.predict(X)
+        return metrics.roc_auc_score(dataset.y, y_proba)
+
+    return PERFORMANCE_METRICS[metric]["function"](dataset.y, model.predict(X))
+
+
+def _compute_gap(result: dict, metric: str) -> float:
+    if PERFORMANCE_METRICS[metric]["is_lower_better"]:
+        return result[metric.upper()][-1] - result[metric.upper()][0]
+
+    return result[metric.upper()][0] - result[metric.upper()][-1]
+
+
+def _combine_results(results: List[dict]):
+    final_results = defaultdict(list)
+
+    # Interleave rows from each dictionary
+    for i in range(len(results[0]["Perturbation Size"])):
+        for result in results:
+            for key in result.keys():
+                final_results[key].append(result[key][i])
+
+    return pd.DataFrame(final_results)
+
+
+def _plot_robustness(
+    results: pd.DataFrame, metric: str, threshold: float, columns: List[str]
+):
+    fig, ax = plt.subplots()
+
+    pallete = sns.color_palette("muted", len(results["Dataset"].unique()))
+    sns.lineplot(
+        data=results,
+        x="Perturbation Size",
+        y=metric.upper(),
+        hue="Dataset",
+        style="Dataset",
+        linewidth=3,
+        markers=True,
+        markersize=10,
+        dashes=False,
+        palette=pallete,
+        ax=ax,
+    )
+
+    if PERFORMANCE_METRICS[metric]["is_lower_better"]:
+        y_label = f"{metric.upper()} (lower is better)"
+    else:
+        threshold = -threshold
+        y_label = f"{metric.upper()} (higher is better)"
+
+    # add dotted threshold line
+    for i in range(len(results["Dataset"].unique())):
+        baseline = results[results["Dataset"] == results["Dataset"].unique()[i]][
+            metric.upper()
+        ].iloc[0]
+        ax.axhline(
+            y=baseline + threshold,
+            color=pallete[i],
+            linestyle="dotted",
+        )
 
-        # Validate list of features columns need to be perterubed
-        if "features_columns" not in self.params:
-            raise ValueError("features_columns must be provided in params")
+    ax.tick_params(axis="x")
+    ax.set_ylabel(y_label, weight="bold", fontsize=18)
+    ax.legend(fontsize=18)
+    ax.set_xlabel(
+        "Perturbation Size (X * Standard Deviation)", weight="bold", fontsize=18
+    )
+    ax.set_title(
+        f"Perturbed Features: {', '.join(columns)}",
+        weight="bold",
+        fontsize=20,
+        wrap=True,
+    )
+
+    # prevent the figure from being displayed
+    plt.close("all")
+
+    return fig
+
+
+# TODO: make this a functional test instead of class-based when appropriate
+# simply have to remove the class and rename this func to OverfitDiagnosis
+def robustness_diagnosis(
+    model: VMModel,
+    datasets: List[VMDataset],
+    metric: str = None,
+    scaling_factor_std_dev_list: List[float] = DEFAULT_STD_DEV_LIST,
+    performance_decay_threshold: float = DEFAULT_DECAY_THRESHOLD,
+):
+    if not metric:
+        metric = (
+            DEFAULT_CLASSIFICATION_METRIC
+            if datasets[0].probability_column(model)
+            else DEFAULT_REGRESSION_METRIC
+        )
+        logger.info(f"Using default metric ({metric.upper()}) for robustness diagnosis")
 
-        features_list = self.params["features_columns"]
-        if features_list is None:
-            features_list = self.inputs.datasets[0].feature_columns
+    if id(scaling_factor_std_dev_list) == id(DEFAULT_STD_DEV_LIST):
+        logger.info(
+            f"Using default scaling factors for the standard deviation of the noise: {DEFAULT_STD_DEV_LIST}"
+        )
 
-        # Check if all elements from features_list are present in the numerical feature columns
-        all_present = all(
-            elem in self.inputs.datasets[0].feature_columns for elem in features_list
+    if id(performance_decay_threshold) == id(DEFAULT_DECAY_THRESHOLD):
+        logger.info(
+            f"Using default performance decay threshold of {DEFAULT_DECAY_THRESHOLD}"
         )
-        if not all_present:
-            raise ValueError(
-                "The list of feature columns provided do not match with training "
-                + "dataset numerical feature columns"
+
+    results = [{} for _ in range(len(datasets))]
+
+    # add baseline results (no perturbation)
+    for dataset, result in zip(datasets, results):
+        result["Perturbation Size"] = [0.0]
+        result["Dataset"] = [f"{dataset.input_id}"]
+        result["Row Count"] = [dataset._df.shape[0]]
+
+        result[metric.upper()] = [
+            _compute_metric(
+                dataset=dataset,
+                model=model,
+                X=dataset.x_df(),
+                metric=metric,
             )
+        ]
+        result["Performance Decay"] = [0.0]
+        result["Passed"] = [True]
+
+    # Iterate scaling factor for the standard deviation list
+    for x_std_dev in scaling_factor_std_dev_list:
+        for dataset, result in zip(datasets, results):
+
+            result["Perturbation Size"].append(x_std_dev)
+            result["Dataset"].append(result["Dataset"][0])
+            result["Row Count"].append(result["Row Count"][0])
+
+            temp_df = dataset.x_df().copy()
+            for feature in dataset.feature_columns_numeric:
+                temp_df[feature] = _add_noise_std_dev(
+                    values=temp_df[feature].to_list(),
+                    x_std_dev=x_std_dev,
+                )
 
-        if self.inputs.datasets[0].text_column in features_list:
-            raise ValueError(
-                "Skiping Robustness Diagnosis test for the dataset with text column"
+            result[metric.upper()].append(
+                _compute_metric(
+                    dataset=dataset,
+                    model=model,
+                    X=temp_df,
+                    metric=metric,
+                )
+            )
+            result["Performance Decay"].append(_compute_gap(result, metric))
+            result["Passed"].append(
+                result["Performance Decay"][-1] < performance_decay_threshold
             )
 
-        train_df = self.inputs.datasets[0].x_df().copy()
-        train_y_true = self.inputs.datasets[0].y
+    results_df = _combine_results(results)
+    fig = _plot_robustness(
+        results=results_df,
+        metric=metric,
+        threshold=performance_decay_threshold,
+        columns=datasets[0].feature_columns_numeric,
+    )
 
-        test_df = self.inputs.datasets[1].x_df().copy()
-        test_y_true = self.inputs.datasets[1].y
+    # rename perturbation size for baseline
+    results_df["Perturbation Size"][
+        results_df["Perturbation Size"] == 0.0
+    ] = "Baseline (0.0)"
 
-        test_results = []
-        test_figures = []
+    return results_df, fig
 
-        results_headers = ["Perturbation Size", "Dataset Type", "Records", "AUC"]
-        results = {k: [] for k in results_headers}
-        # Iterate scaling factor for the standard deviation list
-        for x_std_dev in x_std_dev_list:
-            temp_train_df = train_df.copy()
-            temp_test_df = test_df.copy()
 
-            # Add noise to numeric features columns provided by user
-            for feature in features_list:
-                temp_train_df[feature] = self._add_noise_std_dev(
-                    temp_train_df[feature].to_list(), x_std_dev
-                )
-                temp_test_df[feature] = self._add_noise_std_dev(
-                    temp_test_df[feature].to_list(), x_std_dev
-                )
+@dataclass
+class RobustnessDiagnosis(ThresholdTest):
+    """Evaluate the robustness of a machine learning model to noise
+
+    Robustness refers to a model's ability to maintain a high level of performance in
+    the face of perturbations or changes (particularly noise) added to its input data.
+    This test is designed to help gauge how well the model can handle potential real-
+    world scenarios where the input data might be incomplete or corrupted.
+
+    ## Test Methodology
+    This test is conducted by adding Gaussian noise, proportional to a particular standard
+    deviation scale, to numeric input features of the input datasets. The model's
+    performance on the perturbed data is then evaluated using a user-defined metric or the
+    default metric of AUC for classification tasks and MSE for regression tasks. The results
+    are then plotted to visualize the model's performance decay as the perturbation size
+    increases.
+
+    When using this test, it is highly recommended to tailor the performance metric, list
+    of scaling factors for the standard deviation of the noise, and the performance decay
+    threshold to the specific use case of the model being evaluated.
+
+    **Inputs**:
+    - model (VMModel): The trained model to be evaluated.
+    - datasets (List[VMDataset]): A list of datasets to evaluate the model against.
+
+    ## Parameters
+    - metric (str, optional): The performance metric to be used for evaluation. If not
+        provided, the default metric is used based on the task of the model. Default values
+        are "auc" for classification tasks and "mse" for regression tasks.
+    - scaling_factor_std_dev_list (List[float], optional): A list of scaling factors for
+        the standard deviation of the noise to be added to the input features. The default
+        values are [0.1, 0.2, 0.3, 0.4, 0.5].
+    - performance_decay_threshold (float, optional): The threshold for the performance
+        decay of the model. The default value is 0.05.
+    """
 
-            self._compute_metrics(
-                results, temp_train_df, train_y_true, x_std_dev, "Training"
-            )
-            self._compute_metrics(results, temp_test_df, test_y_true, x_std_dev, "Test")
-
-        fig, df = self._plot_robustness(results, features_list)
-
-        test_figures.append(
-            Figure(
-                for_object=self,
-                key=f"{self.name}:auc",
-                figure=fig,
-                metadata={
-                    "metric": "AUC",
-                    "features_list": features_list,
-                },
-            )
-        )
+    name = "robustness"
+    required_inputs = ["model", "datasets"]
+    default_params = {
+        "metric": None,
+        "scaling_factor_std_dev_list": DEFAULT_STD_DEV_LIST,
+        "performance_decay_threshold": DEFAULT_DECAY_THRESHOLD,
+    }
+    tasks = ["classification", "regression"]
+    tags = [
+        "sklearn",
+        "model_diagnosis",
+        "visualization",
+    ]
 
-        train_auc = df.loc[(df["Dataset Type"] == "Training"), "AUC"].values[0]
-        test_auc = df.loc[(df["Dataset Type"] == "Test"), "AUC"].values[0]
-
-        df["Passed"] = np.where(
-            (df["Dataset Type"] == "Training")
-            & (df["AUC"] >= (train_auc - auc_threshold)),
-            True,
-            np.where(
-                (df["Dataset Type"] == "Test")
-                & (df["AUC"] >= (test_auc - auc_threshold)),
-                True,
-                False,
-            ),
-        )
-        test_results.append(
-            ThresholdTestResult(
-                test_name="AUC",
-                column=features_list,
-                passed=True,
-                values={"records": df.to_dict("records")},
-            )
+    def run(self):
+        results, fig = robustness_diagnosis(
+            model=self.inputs.model,
+            datasets=self.inputs.datasets,
+            metric=self.params["metric"],
+            scaling_factor_std_dev_list=self.params["scaling_factor_std_dev_list"],
+            performance_decay_threshold=self.params["performance_decay_threshold"],
         )
+
         return self.cache_results(
-            test_results, passed=df["Passed"].all(), figures=test_figures
+            passed=results["Passed"].all(),
+            test_results_list=[
+                ThresholdTestResult(
+                    test_name=self.params["metric"],
+                    passed=results["Passed"].all(),
+                    values=results.to_dict(orient="records"),
+                )
+            ],
+            figures=[
+                Figure(
+                    for_object=self,
+                    key=f"{self.name}:{self.params['metric']}",
+                    figure=fig,
+                )
+            ],
         )
 
     def summary(self, results: List[ThresholdTestResult], _):
-        results_table = [
-            record for result in results for record in result.values["records"]
-        ]
         return ResultSummary(
             results=[
                 ResultTable(
-                    data=results_table,
-                    metadata=ResultTableMetadata(title="Robustness test"),
+                    data=results[0].values,
+                    metadata=ResultTableMetadata(title="Robustness Diagnosis Results"),
                 )
             ]
         )
 
-    def _compute_metrics(
-        self,
-        results: dict,
-        df: pd.DataFrame,
-        y_true: str,
-        x_std_dev: float,
-        dataset_type: str,
-    ):
-        """
-        Compute evaluation metrics for a given perturbed dataset.
-        Args:
-        results (dict): A dictionary to store the results of the computation.
-        df (pd.DataFrame): A Pandas dataframe containing the dataset to evaluate.
-        y_true (str): A string representing the name of the column containing the true target values.
-        x_std_dev (float): A float representing the standard deviation of the perturbation applied to the dataset.
-        dataset_type (str): A string representing the type of dataset (e.g. "training", "validation", "test").
-        Returns:
-        None
-        """
-        results["Dataset Type"].append(dataset_type)
-        results["Perturbation Size"].append(x_std_dev)
-        results["Records"].append(df.shape[0])
-
-        try:
-            y_proba = self.inputs.model.predict_proba(df)
-        except MissingOrInvalidModelPredictFnError:
-            y_proba = self.inputs.model.predict(df)
-
-        results["AUC"].append(metrics.roc_auc_score(y_true, y_proba))
-
-    def _add_noise_std_dev(
-        self, values: List[float], x_std_dev: float
-    ) -> Tuple[List[float], float]:
-        """
-        Adds Gaussian noise to a list of values.
-        Args:
-            values (list[float]): A list of numerical values to which noise is added.
-            x_std_dev (float): A scaling factor for the standard deviation of the noise.
-        Returns:
-            tuple[list[float], float]: A tuple containing:
-                - A list of noisy values, where each value is the sum of the corresponding value
-                in the input list and a randomly generated value sampled from a Gaussian distribution
-                with mean 0 and standard deviation x_std_dev times the standard deviation of the input list.
-                - The standard deviation of the input list of values.
-        """
-        std_dev = np.std(values)
-        noise_list = np.random.normal(0, x_std_dev * std_dev, size=len(values))
-        noisy_values = list(map(add, noise_list, values))
-
-        return noisy_values
-
-    def _plot_robustness(self, results: dict, features_columns: List[str]):
-        """
-        Plots the model's auc under feature perturbations.
-        Args:
-            results (dict): A dictionary containing the results of the evaluation.
-                It has the following keys:
-                    - 'Dataset Type': the type of dataset evaluated, e.g. 'Training' or 'Test'.
-                    - 'Perturbation Size': the size of the perturbation applied to the features.
-                    - 'Records': the number of records evaluated.
-                    - 'auc': the ROC AUC score obtained for the evaluation.
-                The values of each key are lists containing the results for each evaluation.
-            features_columns (list[str]): A list containing the names of the features perturbed.
-        Returns:
-            tuple[matplotlib.figure.Figure, pd.DataFrame]: A tuple containing the matplotlib Figure object
-            and a DataFrame containing the results used to generate the plot.
-        """
-        df = pd.DataFrame(results)
-
-        # Create a bar plot using seaborn library
-        fig, ax = plt.subplots()
-        sns.lineplot(
-            data=df,
-            x="Perturbation Size",
-            y="AUC",
-            hue="Dataset Type",
-            style="Dataset Type",
-            linewidth=3,
-            markers=True,
-            markersize=10,
-            dashes=False,
-            palette=["red", "blue"],
-            ax=ax,
-        )
-        ax.tick_params(axis="x")
-        ax.set_ylabel("AUC", weight="bold", fontsize=18)
-        ax.legend(fontsize=18)
-        ax.set_xlabel(
-            "Perturbation Size (X * Standard Deviation)", weight="bold", fontsize=18
-        )
-        ax.set_title(
-            f"Perturbed Features: {', '.join(features_columns)}",
-            weight="bold",
-            fontsize=20,
-            wrap=True,
-        )
-
-        # Do this if you want to prevent the figure from being displayed
-        plt.close("all")
-
-        # fig, ax = plt.subplots()
-        return fig, df
-
     def test(self):
         """Unit Test for Robustness Diagnosis Threshold Test"""
         # Verify the result object is present
@@ -313,16 +372,8 @@ class RobustnessDiagnosis(ThresholdTest):
         # Verify test results and their type
         assert isinstance(self.result.test_results.results, list)
 
-        # Check for presence and validity of 'values' dict and 'passed' flag in each result
+        # Check for presence and validity of 'values' and 'passed' flag in each result
         for test_result in self.result.test_results.results:
             assert "values" in test_result.__dict__
             assert "passed" in test_result.__dict__
-            assert isinstance(test_result.values, dict)
-            assert "records" in test_result.values
-
-            # For unperturbed training dataset, auc should be present
-            if (
-                test_result.column == self.params["features_columns"]
-                and 0.0 in test_result.values["records"][0]["Perturbation Size"]
-            ):
-                assert "AUC" in test_result.values["records"][0]
+            assert isinstance(test_result.values, list)

validmind/tests/run.py

@@ -405,7 +405,7 @@ def run_test(
 
     if unit_metrics:
         metric_id_name = "".join(word[0].upper() + word[1:] for word in name.split())
-        test_id = f"validmind.composite_test.{metric_id_name}"
+        test_id = f"validmind.composite_metric.{metric_id_name}"
 
         error, TestClass = load_composite_metric(
             unit_metrics=unit_metrics, metric_name=metric_id_name

validmind/vm_models/dataset/utils.py

@@ -10,6 +10,7 @@ import pandas as pd
 
 from validmind.errors import MissingOrInvalidModelPredictFnError
 from validmind.logging import get_logger
+from validmind.vm_models.model import ModelTask
 
 logger = get_logger(__name__)
 
@@ -118,8 +119,14 @@ def compute_predictions(model, X, **kwargs) -> tuple:
             "You can pass `prediction_values` or `prediction_columns` to use precomputed predictions"
         )
 
-    # TODO: this is really not ideal/robust and should not be handled by dataset class
-    if probability_values is None and _is_probabilties(prediction_values):
+    if model.attributes.task is ModelTask.REGRESSION:
+        logger.info("Model is configured for regression.")
+        return probability_values, prediction_values
+
+    if probability_values is None and (
+        model.attributes.task is ModelTask.CLASSIFICATION
+        or _is_probabilties(prediction_values)
+    ):
         logger.info(
             "Predict method returned probabilities instead of direct labels or regression values. "
             "This implies the model is likely configured for a classification task with probability output."

validmind/vm_models/model.py

@@ -9,6 +9,7 @@ import importlib
 import inspect
 from abc import abstractmethod
 from dataclasses import dataclass
+from enum import Enum
 
 from validmind.errors import MissingOrInvalidModelPredictFnError
 
@@ -38,6 +39,14 @@ R_MODEL_METHODS = [
 ]
 
 
+class ModelTask(Enum):
+    """Model task enums"""
+
+    # TODO: add more tasks
+    CLASSIFICATION = "classification"
+    REGRESSION = "regression"
+
+
 class ModelPipeline:
     """Helper class for chaining models together
 
@@ -65,6 +74,7 @@ class ModelAttributes:
     framework: str = None
     framework_version: str = None
     language: str = None
+    task: ModelTask = None
 
     @classmethod
     def from_dict(cls, data):
@@ -76,6 +86,7 @@ class ModelAttributes:
             framework=data.get("framework"),
             framework_version=data.get("framework_version"),
             language=data.get("language"),
+            task=ModelTask(data.get("task")) if data.get("task") else None,
         )
 
 
@@ -108,7 +119,7 @@ class VMModel(VMInput):
 
         self.name = name or self.__class__.__name__
 
-        self.attributes = attributes
+        self.attributes = attributes or ModelAttributes()
 
         # set any additional attributes passed in (likely for subclasses)
         for key, value in kwargs.items():

{validmind-2.5.1.dist-info → validmind-2.5.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.5.1
+Version: 2.5.6
 Summary: ValidMind Developer Framework
 License: Commercial License
 Author: Andres Rodriguez
@@ -49,7 +49,7 @@ Requires-Dist: scorecardpy (>=0.1.9.6,<0.2.0.0)
 Requires-Dist: seaborn
 Requires-Dist: sentencepiece (>=0.2.0,<0.3.0) ; extra == "all" or extra == "huggingface" or extra == "llm"
 Requires-Dist: sentry-sdk (>=1.24.0,<2.0.0)
-Requires-Dist: shap (>=0.42.0,<0.43.0)
+Requires-Dist: shap (==0.44.1)
 Requires-Dist: statsmodels
 Requires-Dist: tabulate (>=0.8.9,<0.9.0)
 Requires-Dist: textblob (>=0.18.0.post0,<0.19.0)

{validmind-2.5.1.dist-info → validmind-2.5.6.dist-info}/RECORD

@@ -1,9 +1,9 @@
 validmind/__init__.py,sha256=UfmzPwUCdUWbWq3zPqqmq4jw0_kfl3hX4U72p_seE4I,3700
-validmind/__version__.py,sha256=PfQ9ThOuZlUZhThya-_PpR02LjazRR6LNSivpta03mM,22
+validmind/__version__.py,sha256=1X5kk-wx4BPu-LJgtjkfia2ZtDoOHOAsSyEsLKKQCY0,22
 validmind/ai/test_descriptions.py,sha256=Q1Ftus4x5eiVLKWJu7hqPLukBQZzhy-dARqq_6_JWtk,9464
 validmind/ai/utils.py,sha256=TEXII_S5CpkpczzSyHwTlqLcPMLnPBJWEBR6QFMKh1U,3421
 validmind/api_client.py,sha256=JZIJWuYtvl-VEVi_AK4c839Fn7cGa40J2d4_4FUZcno,17483
-validmind/client.py,sha256=guXu_9um4caPpepbAsfKgjLc63Ygx07Lgp8wZJD3p6Y,18653
+validmind/client.py,sha256=tFqjbTbJ5AVOythRMn5vcoBm3uCKFbV2yPmk-XqForE,18902
 validmind/client_config.py,sha256=58L6s6-9vFWC9vkSs_98CjV1YWmlksdhblJtPQxQsAk,1611
 validmind/datasets/__init__.py,sha256=oYfcvW7BAyUgpghBOnTeGbQF6tpFAWg38rRirdLr8m8,262
 validmind/datasets/classification/__init__.py,sha256=HlTOBLyb6IorRYmAhP3AIyX-l-NyemyDjV8BBOdrCrY,1787
@@ -86,7 +86,7 @@ validmind/test_suites/tabular_datasets.py,sha256=WE4eLzRCfiqAxRqXnZFRR3Lo_u-TI6K
 validmind/test_suites/text_data.py,sha256=YGVGBB05356jN9Gzcy5CHShRzo1fm5mKsZY7YBq0cYU,739
 validmind/test_suites/time_series.py,sha256=msUyYySAe5VHJJp6z0k0cNt2ekMB8-XkxGER75Zs1hs,6724
 validmind/tests/__init__.py,sha256=niYvgTHmjS5E42mJMCrzq1vP8PTKCWxVsqSkAaw2wsE,1036
-validmind/tests/__types__.py,sha256=AaPsQrxikIasGshJN5AmKCTzLaZ9d4QBDT1c0Br2sDE,10142
+validmind/tests/__types__.py,sha256=Kgxiyf2djYcKl3ZMg3ND1_f1Hd7Z9VeRTEUnePDDf0U,10085
 validmind/tests/_store.py,sha256=G604L9g-XIJz8u7BLbHVVVcbx96tDYjAAciaF7wJoiM,2743
 validmind/tests/data_validation/ACFandPACFPlot.py,sha256=NLoLe-9Z6_41RBee-gRYe4u3kaGojF7ujlyyIk4o3BU,4900
 validmind/tests/data_validation/ADF.py,sha256=36ZdB8L-hgN0EnYlcxeSsQ3luWip8Qfz_nrYV-1lr74,5113
@@ -190,15 +190,14 @@ validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py,sha2
 validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py,sha256=npnOPAoXb5FoiwQEwp_gDcbGa5xk4rYnXChTJnuGX64,4405
 validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py,sha256=qLydyTQ6mzHOYQzqysjPPe_ltiTsRfPEhZDEDm5XxX8,4825
 validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py,sha256=ereo_dGf19xqvOGz7zcGwhDRU_UqvjFRi5n4KmGDKl8,4511
-validmind/tests/model_validation/ragas/AnswerCorrectness.py,sha256=XOEpsVqeaUCGUILu81ZLXSDlyqLStPX9ZKUJDrTh2Bg,5138
-validmind/tests/model_validation/ragas/AnswerRelevance.py,sha256=Tz3rNO2PnHvvAw7YKWPUZiiitJTH6VJDG8LjC9Nkwx8,4997
-validmind/tests/model_validation/ragas/AnswerSimilarity.py,sha256=O1xq5GNnWB9roscmaBRGiWEi0BvXh705ppeKJTP3O58,4457
-validmind/tests/model_validation/ragas/AspectCritique.py,sha256=08jlfL4qVuRM-U4Y-zGYb9iy-DLsXk7JtdvA0yRK498,6263
-validmind/tests/model_validation/ragas/ContextEntityRecall.py,sha256=Du3A5Jkpt9_msaF3bDy6tNvgomUkmgwsIxO2zdZmmyA,4904
-validmind/tests/model_validation/ragas/ContextPrecision.py,sha256=-4LBSu1ovzIuf2evSoSwyYdrNtGl5-9w8yRQzjcdDtY,4630
-validmind/tests/model_validation/ragas/ContextRecall.py,sha256=i72mPF8eO8BUrshdvn6Mpoq9oFSfvPH1lfWk-LKvS8w,4567
-validmind/tests/model_validation/ragas/ContextRelevancy.py,sha256=TcfC-O7vj2zDU1UqIYC4KgAQaA9aaOtSUHuXfl70JJE,4155
-validmind/tests/model_validation/ragas/Faithfulness.py,sha256=89EeM0lrUq5MAhKYhOO9cnp32WCap6eG2n28SjZH9c4,4525
+validmind/tests/model_validation/ragas/AnswerCorrectness.py,sha256=UhspG4nY901ZhAmgEzABWiYQPx0rKEJqQnrFCunwnN8,5139
+validmind/tests/model_validation/ragas/AnswerRelevance.py,sha256=_hD24Ecs1TZQl-lEoFtdgNGg3hXL-VyfmimiJaovnvY,4998
+validmind/tests/model_validation/ragas/AnswerSimilarity.py,sha256=c1xc4F4gwrrJKn1eEhZQbw1nc39Q2zS75AS9G3XUMAI,4458
+validmind/tests/model_validation/ragas/AspectCritique.py,sha256=2hGGVMb0_va9Gjqyu1OUI-CSpD6k7ICMnwEYEtRGadk,6264
+validmind/tests/model_validation/ragas/ContextEntityRecall.py,sha256=zZGenHhWZQRm9CxAl-ZgbHva6vUlbI_jsFkuY4B2LS8,4905
+validmind/tests/model_validation/ragas/ContextPrecision.py,sha256=FdXTL8KXv6q5lR1BItkCAt105qikYmd89KgvLOqkatE,4631
+validmind/tests/model_validation/ragas/ContextRecall.py,sha256=fNawcRi5M8773mh-QcuUaJsdoLrkCDPza-qvOBMApKk,4568
+validmind/tests/model_validation/ragas/Faithfulness.py,sha256=Jg9SK9NPSbLG9nmM1tu55FGSFpqbb3P4e7kPg20OD_8,4526
 validmind/tests/model_validation/ragas/utils.py,sha256=zh9_pGitutGBS4Tvk3Bw1D-QVnDueggNErAhAvMPUOA,3130
 validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py,sha256=KhQroHKDG4gpRAoD9Clw65qNslwGS93rTETdZTOqoTk,2840
 validmind/tests/model_validation/sklearn/AdjustedRandIndex.py,sha256=nSs1BGC8MSWWp7T6M0FZR5kNPraiZwRJF7U6LNCoMMM,2715
@@ -217,7 +216,7 @@ validmind/tests/model_validation/sklearn/MinimumAccuracy.py,sha256=5KSAd29dbKs3n
 validmind/tests/model_validation/sklearn/MinimumF1Score.py,sha256=TaLHk98CwQigyt17L1uBBLC25D5J_IKb6a_IFJFO7AE,4618
 validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py,sha256=Z5JZ4edtzuyneI8qSmGv-OKL2PVq5dg44CwSmePz3OU,5102
 validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.py,sha256=-fGgddsc_0832zTl_gRRsLx2sZWBPB0FdS5YmbluN8s,6132
-validmind/tests/model_validation/sklearn/OverfitDiagnosis.py,sha256=1nei-MIF7utxuPJXnhTKIaLugUsxk1s4cFob1CR08Yg,13444
+validmind/tests/model_validation/sklearn/OverfitDiagnosis.py,sha256=kB392ZQYqsPCgVDbqZ-056PliVJ_3Txogf-5iF37qgI,12750
 validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py,sha256=CqMuBuNWzzTtzVcmhAlJHPmtyDO5YuaoXk5hhIXmRuY,4926
 validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py,sha256=chYVS4OcvSG3YA91N7VDJ4Lh7EDgNEcUM8_k72s13IM,10072
 validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py,sha256=V0SS06u8DsyaJpL0S14HBPAQwJJYXnvP3fNp2P4CT84,4363
@@ -227,7 +226,7 @@ validmind/tests/model_validation/sklearn/RegressionErrorsComparison.py,sha256=CH
 validmind/tests/model_validation/sklearn/RegressionModelsPerformanceComparison.py,sha256=ELYhY_My1YqS4_i2fnHgL5Dg7vKUIa0wska0bkAFkuU,5737
 validmind/tests/model_validation/sklearn/RegressionR2Square.py,sha256=Ojm5sz3re4rk17u7xiezn1P_rp7wcA3etKgzdhGYH-s,4906
 validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.py,sha256=tGJKpfeTvU2xBxsYbQSC5GPDcCS2_j0FcT3uceXZduI,2761
-validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py,sha256=ntzu5W4JF6pM8YXwtWVnQHx8zGaZMYSTlQlD38XpCUo,13366
+validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py,sha256=KXBQ5-3ZDLil3WBZR-zWySelN_zb5Ob4Uvuoi1qfFaI,12821
 validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py,sha256=ECYjHHIz5kfnLi2XlzWOKquRf23_77kdcPK8Xw2qwQk,8887
 validmind/tests/model_validation/sklearn/SilhouettePlot.py,sha256=6PZ_sqiPBpL4_fyRE_sg0bSWWrDkryh_v-88KK4i3RQ,6185
 validmind/tests/model_validation/sklearn/TrainingTestDegradation.py,sha256=K3F8Ev7nIaIjwLHC9ljnMp07YwZeqo4RLui5C6IDuR8,7209
@@ -270,7 +269,7 @@ validmind/tests/prompt_validation/Robustness.py,sha256=fBdkYnO9yoBazz4wD-l62tT8D
 validmind/tests/prompt_validation/Specificity.py,sha256=h3gKRTTi2rfnGWmGC1YnSt2s_VbZU4KX0iY7LciZ3PU,6068
 validmind/tests/prompt_validation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 validmind/tests/prompt_validation/ai_powered_test.py,sha256=7TTeIR5GotQosm7oVT8Y3KnwPB3XkVT1Fzhckpr-SgE,1963
-validmind/tests/run.py,sha256=WuLV8iY2xN7bRPu5px75-rgRKeh_XYPtbdLhqG8Dugo,15874
+validmind/tests/run.py,sha256=K_EiaquuSv7rVnr-wl2uO1HUDypWGIEXA8JIjw5xsKw,15876
 validmind/tests/test_providers.py,sha256=47xe5eb5ufvj1jmhdRsbSvDQTXSDpFDFNeXg3xtXwhw,5320
 validmind/tests/utils.py,sha256=kNrxfUYbj4DwmkZtpp_1rG4GMUGxYEhvqnYR_A7qAKM,471
 validmind/unit_metrics/__init__.py,sha256=mFk52eU7bOQKTpruKSrPyzjmxFUpIi5RZuwIE5BVFHU,7345
@@ -295,10 +294,10 @@ validmind/utils.py,sha256=DYUB3nig6MJwct5dymhy7Gt9apwzPVipKAWxSrm0-tg,15849
 validmind/vm_models/__init__.py,sha256=V5DH-E1Rkvl-HQEkilppVCHBag9MQXkzyoORLW3LSGQ,1210
 validmind/vm_models/dataset/__init__.py,sha256=U4CxZjdoc0dd9u2AqBl5PJh1UVbzXWNrmundmjLF-qE,346
 validmind/vm_models/dataset/dataset.py,sha256=hBaczQjo-Jb1u6Ma5yX86m6JzT16XndAlq32WbHqVx8,25645
-validmind/vm_models/dataset/utils.py,sha256=DRFCg93YE7sTRrWAGt1RIyvzPjINagMk6zUw7z692d0,5325
+validmind/vm_models/dataset/utils.py,sha256=VMcPEgwW9oW5D0MCa_MqXCq_sEzzsLLRmS4RaYrsif0,5530
 validmind/vm_models/figure.py,sha256=iSrvPcCG5sQrMkX1Fh6c5utRzaroh3bc6IlnGDOK_Eg,6651
 validmind/vm_models/input.py,sha256=qLdqz_bktr4v0YcPha2vFdDvmkC-btT1pH9zBIkt1OY,1046
-validmind/vm_models/model.py,sha256=P-zKbh0TrU_4ZK-bA0l83h6K6nfU6v0lIpC4mfCl6Fw,6115
+validmind/vm_models/model.py,sha256=Dewux_jTgUAXPgHW6ZtJTa8WvH0WkWsryO43DI9HkMU,6409
 validmind/vm_models/test/metric.py,sha256=DvXMju36JzxArXNWimq3SSrSUoIHkyvDbuhbgBOKxkk,3357
 validmind/vm_models/test/metric_result.py,sha256=Bak4GDrMlNq5NtgP5exwlPsKZgz3tWgtC6jZqtHjvqM,1987
 validmind/vm_models/test/output_template.py,sha256=njqCAMyLxwadkCWhACVskyL9-psTgmUysaeeirTVAX4,1500
@@ -312,8 +311,8 @@ validmind/vm_models/test_suite/runner.py,sha256=aewxadRfoOPH48jes2Gtb3Ju_FWFfVM_
 validmind/vm_models/test_suite/summary.py,sha256=GQRNe2ZvvqjQN0yKmaN7ohAUjRFQIN4YYUYxfOuWN6M,4682
 validmind/vm_models/test_suite/test.py,sha256=_GfbK36l98SjzgVcucmp0OKBJKqMW3neO7SqJ3EWeps,5049
 validmind/vm_models/test_suite/test_suite.py,sha256=Cns2wL54v0T5Mv5_HJb3kMeaa4rtycdqT8KxK9_rWEU,6279
-validmind-2.5.1.dist-info/LICENSE,sha256=XonPUfwjvrC5Ombl3y-ko0Wubb1xdG_7nzvIbkZRKHw,35772
-validmind-2.5.1.dist-info/METADATA,sha256=ZnwdazSYewQA2IZLaVIUQkuxmDl6yVke6kjyiSkLEJk,4250
-validmind-2.5.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-validmind-2.5.1.dist-info/entry_points.txt,sha256=HuW7YyOv9u_OEWpViQXtv0nfoI67uieJHawKWA4Hv9A,76
-validmind-2.5.1.dist-info/RECORD,,
+validmind-2.5.6.dist-info/LICENSE,sha256=XonPUfwjvrC5Ombl3y-ko0Wubb1xdG_7nzvIbkZRKHw,35772
+validmind-2.5.6.dist-info/METADATA,sha256=cPAO_Hlc8esuZm1W96GFv09amk1VNxn7Oh33iUPUfbI,4242
+validmind-2.5.6.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+validmind-2.5.6.dist-info/entry_points.txt,sha256=HuW7YyOv9u_OEWpViQXtv0nfoI67uieJHawKWA4Hv9A,76
+validmind-2.5.6.dist-info/RECORD,,

validmind/tests/model_validation/ragas/ContextRelevancy.py

@@ -1,119 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-import warnings
-
-import plotly.express as px
-from datasets import Dataset
-
-from validmind import tags, tasks
-
-from .utils import get_ragas_config, get_renamed_columns
-
-
-@tags("ragas", "llm", "retrieval_performance")
-@tasks("text_qa", "text_generation", "text_summarization", "text_classification")
-def ContextRelevancy(
-    dataset,
-    question_column: str = "question",
-    contexts_column: str = "contexts",
-):
-    """
-    Evaluates the context relevancy metric for entries in a dataset and visualizes the
-    results.
-
-    This metric gauges the relevancy of the retrieved context, calculated based on both
-    the `question` and `contexts`. The values fall within the range of (0, 1), with
-    higher values indicating better relevancy.
-
-    Ideally, the retrieved context should exclusively contain essential information to
-    address the provided query. To compute this, we initially estimate the value of by
-    identifying sentences within the retrieved context that are relevant for answering
-    the given question. The final score is determined by the following formula:
-
-    $$
-    \\text{context relevancy} = {|S| \\over |\\text{Total number of sentences in retrieved context}|}
-    $$
-
-    ### Configuring Columns
-
-    This metric requires the following columns in your dataset:
-    - `question` (str): The text query that was input into the model.
-    - `contexts` (List[str]): A list of text contexts which are retrieved and which
-    will be evaluated to make sure they are relevant to the question.
-
-    If the above data is not in the appropriate column, you can specify different column
-    names for these fields using the parameters `question_column` and `contexts_column`.
-
-    For example, if your dataset has this data stored in different columns, you can
-    pass the following parameters:
-    ```python
-    {
-        "question_column": "question",
-        "contexts_column": "context_info"
-    }
-    ```
-
-    If the data is stored as a dictionary in another column, specify the column and key
-    like this:
-    ```python
-    pred_col = dataset.prediction_column(model)
-    params = {
-        "contexts_column": f"{pred_col}.contexts",
-    }
-    ```
-
-    For more complex situations, you can use a function to extract the data:
-    ```python
-    pred_col = dataset.prediction_column(model)
-    params = {
-        "contexts_column": lambda x: [x[pred_col]["context_message"]],
-    }
-    ```
-    """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import context_relevancy
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
-    warnings.filterwarnings(
-        "ignore",
-        category=FutureWarning,
-        message="promote has been superseded by promote_options='default'.",
-    )
-
-    required_columns = {
-        "question": question_column,
-        "contexts": contexts_column,
-    }
-
-    df = get_renamed_columns(dataset.df, required_columns)
-
-    result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[context_relevancy], **get_ragas_config()
-    ).to_pandas()
-
-    fig_histogram = px.histogram(x=result_df["context_relevancy"].to_list(), nbins=10)
-    fig_box = px.box(x=result_df["context_relevancy"].to_list())
-
-    return (
-        {
-            "Scores (will not be uploaded to UI)": result_df[
-                ["question", "contexts", "context_relevancy"]
-            ],
-            "Aggregate Scores": [
-                {
-                    "Mean Score": result_df["context_relevancy"].mean(),
-                    "Median Score": result_df["context_relevancy"].median(),
-                    "Max Score": result_df["context_relevancy"].max(),
-                    "Min Score": result_df["context_relevancy"].min(),
-                    "Standard Deviation": result_df["context_relevancy"].std(),
-                    "Count": len(result_df),
-                }
-            ],
-        },
-        fig_histogram,
-        fig_box,
-    )