validmind 2.1.1__py3-none-any.whl → 2.2.4__py3-none-any.whl

validmind/tests/model_validation/ragas/ContextRecall.py

@@ -0,0 +1,123 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import warnings
+
+import plotly.express as px
+from datasets import Dataset
+from ragas import evaluate
+from ragas.metrics import context_recall
+
+from validmind import tags, tasks
+
+from .utils import get_renamed_columns
+
+
+@tags("ragas", "llm", "retrieval_performance")
+@tasks("text_qa", "text_generation", "text_summarization", "text_classification")
+def ContextRecall(
+    dataset,
+    question_column: str = "question",
+    contexts_column: str = "contexts",
+    ground_truth_column: str = "ground_truth",
+):
+    """
+    Context recall measures the extent to which the retrieved context aligns with the
+    annotated answer, treated as the ground truth. It is computed based on the `ground
+    truth` and the `retrieved context`, and the values range between 0 and 1, with higher
+    values indicating better performance.
+
+    To estimate context recall from the ground truth answer, each sentence in the ground
+    truth answer is analyzed to determine whether it can be attributed to the retrieved
+    context or not. In an ideal scenario, all sentences in the ground truth answer
+    should be attributable to the retrieved context.
+
+
+    The formula for calculating context recall is as follows:
+    $$
+    \\text{context recall} = {|\\text{GT sentences that can be attributed to context}| \\over |\\text{Number of sentences in GT}|}
+    $$
+
+    ### Configuring Columns
+
+    This metric requires the following columns in your dataset:
+    - `question` (str): The text query that was input into the model.
+    - `contexts` (List[str]): A list of text contexts which are retrieved and which
+    will be evaluated to make sure they contain all items in the ground truth.
+    - `ground_truth` (str): The ground truth text to compare with the retrieved contexts.
+
+    If the above data is not in the appropriate column, you can specify different column
+    names for these fields using the parameters `question_column`, `contexts_column`
+    and `ground_truth_column`.
+
+    For example, if your dataset has this data stored in different columns, you can
+    pass the following parameters:
+    ```python
+    {
+        "question_column": "question",
+        "contexts_column": "context_info"
+        "ground_truth_column": "my_ground_truth_col",
+    }
+    ```
+
+    If the data is stored as a dictionary in another column, specify the column and key
+    like this:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "contexts_column": f"{pred_col}.contexts",
+        "ground_truth_column": "my_ground_truth_col",
+    }
+    ```
+
+    For more complex situations, you can use a function to extract the data:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "contexts_column": lambda x: [x[pred_col]["context_message"]],
+        "ground_truth_column": "my_ground_truth_col",
+    }
+    ```
+    """
+    warnings.filterwarnings(
+        "ignore",
+        category=FutureWarning,
+        message="promote has been superseded by promote_options='default'.",
+    )
+
+    required_columns = {
+        "question": question_column,
+        "contexts": contexts_column,
+        "ground_truth": ground_truth_column,
+    }
+
+    df = get_renamed_columns(dataset.df, required_columns)
+
+    result_df = evaluate(
+        Dataset.from_pandas(df),
+        metrics=[context_recall],
+    ).to_pandas()
+
+    fig_histogram = px.histogram(x=result_df["context_recall"].to_list(), nbins=10)
+    fig_box = px.box(x=result_df["context_recall"].to_list())
+
+    return (
+        {
+            "Scores": result_df[
+                ["question", "contexts", "ground_truth", "context_recall"]
+            ],
+            "Aggregate Scores": [
+                {
+                    "Mean Score": result_df["context_recall"].mean(),
+                    "Median Score": result_df["context_recall"].median(),
+                    "Max Score": result_df["context_recall"].max(),
+                    "Min Score": result_df["context_recall"].min(),
+                    "Standard Deviation": result_df["context_recall"].std(),
+                    "Count": len(result_df),
+                }
+            ],
+        },
+        fig_histogram,
+        fig_box,
+    )

validmind/tests/model_validation/ragas/ContextRelevancy.py

@@ -0,0 +1,114 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import warnings
+
+import plotly.express as px
+from datasets import Dataset
+from ragas import evaluate
+from ragas.metrics import context_relevancy
+
+from validmind import tags, tasks
+
+from .utils import get_renamed_columns
+
+
+@tags("ragas", "llm", "retrieval_performance")
+@tasks("text_qa", "text_generation", "text_summarization", "text_classification")
+def ContextRelevancy(
+    dataset,
+    question_column: str = "question",
+    contexts_column: str = "contexts",
+):
+    """
+    Evaluates the context relevancy metric for entries in a dataset and visualizes the
+    results.
+
+    This metric gauges the relevancy of the retrieved context, calculated based on both
+    the `question` and `contexts`. The values fall within the range of (0, 1), with
+    higher values indicating better relevancy.
+
+    Ideally, the retrieved context should exclusively contain essential information to
+    address the provided query. To compute this, we initially estimate the value of by
+    identifying sentences within the retrieved context that are relevant for answering
+    the given question. The final score is determined by the following formula:
+
+    $$
+    \\text{context relevancy} = {|S| \\over |\\text{Total number of sentences in retrieved context}|}
+    $$
+
+    ### Configuring Columns
+
+    This metric requires the following columns in your dataset:
+    - `question` (str): The text query that was input into the model.
+    - `contexts` (List[str]): A list of text contexts which are retrieved and which
+    will be evaluated to make sure they are relevant to the question.
+
+    If the above data is not in the appropriate column, you can specify different column
+    names for these fields using the parameters `question_column` and `contexts_column`.
+
+    For example, if your dataset has this data stored in different columns, you can
+    pass the following parameters:
+    ```python
+    {
+        "question_column": "question",
+        "contexts_column": "context_info"
+    }
+    ```
+
+    If the data is stored as a dictionary in another column, specify the column and key
+    like this:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "contexts_column": f"{pred_col}.contexts",
+    }
+    ```
+
+    For more complex situations, you can use a function to extract the data:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "contexts_column": lambda x: [x[pred_col]["context_message"]],
+    }
+    ```
+    """
+    warnings.filterwarnings(
+        "ignore",
+        category=FutureWarning,
+        message="promote has been superseded by promote_options='default'.",
+    )
+
+    required_columns = {
+        "question": question_column,
+        "contexts": contexts_column,
+    }
+
+    df = get_renamed_columns(dataset.df, required_columns)
+
+    result_df = evaluate(
+        Dataset.from_pandas(df),
+        metrics=[context_relevancy],
+    ).to_pandas()
+
+    fig_histogram = px.histogram(x=result_df["context_relevancy"].to_list(), nbins=10)
+    fig_box = px.box(x=result_df["context_relevancy"].to_list())
+
+    return (
+        {
+            "Scores": result_df[["question", "contexts", "context_relevancy"]],
+            "Aggregate Scores": [
+                {
+                    "Mean Score": result_df["context_relevancy"].mean(),
+                    "Median Score": result_df["context_relevancy"].median(),
+                    "Max Score": result_df["context_relevancy"].max(),
+                    "Min Score": result_df["context_relevancy"].min(),
+                    "Standard Deviation": result_df["context_relevancy"].std(),
+                    "Count": len(result_df),
+                }
+            ],
+        },
+        fig_histogram,
+        fig_box,
+    )

validmind/tests/model_validation/ragas/Faithfulness.py

@@ -0,0 +1,119 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import warnings
+
+import plotly.express as px
+from datasets import Dataset
+from ragas import evaluate
+from ragas.metrics import faithfulness
+
+from validmind import tags, tasks
+
+from .utils import get_renamed_columns
+
+
+@tags("ragas", "llm", "rag_performance")
+@tasks("text_qa", "text_generation", "text_summarization")
+def Faithfulness(
+    dataset,
+    answer_column="answer",
+    contexts_column="contexts",
+):
+    """
+    Evaluates the faithfulness of the generated answers with respect to retrieved contexts.
+
+    This metric uses a judge LLM to measure the factual consistency of the generated answer
+    against the given context(s). It is calculated using the generated text `answer` from
+    the LLM and the retrieved `contexts` which come from some RAG process. The score is
+    a value between 0 and 1, where a higher score indicates that the generated answer is
+    more faithful to the given context(s).
+
+    The generated answer is regarded as faithful if all the claims that are made in the
+    answer can be inferred from the given context. To calculate this a set of claims from
+    the generated answer is first identified. Then each one of these claims are cross checked
+    with given context to determine if it can be inferred from given context or not. The
+    faithfulness score formula is as follows:
+
+    $$
+    \\text{Faithfulness score} = {|\\text{Number of claims in the generated answer that can be inferred from given context}| \\over |\\text{Total number of claims in the generated answer}|}
+    $$
+
+    ### Configuring Columns
+
+    This metric requires the following columns in your dataset:
+    - `contexts` (List[str]): A list of text contexts which are retrieved to generate
+    the answer.
+    - `answer` (str): The response generated by the model which will be evaluated for
+    faithfulness against the given contexts.
+
+    If the above data is not in the appropriate column, you can specify different column
+    names for these fields using the parameters `contexts_column` and `answer_column`.
+
+    For example, if your dataset has this data stored in different columns, you can
+    pass the following parameters:
+    ```python
+    {
+        "contexts_column": "context_info"
+        "answer_column": "my_answer_col",
+    }
+    ```
+
+    If the data is stored as a dictionary in another column, specify the column and key
+    like this:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "contexts_column": f"{pred_col}.contexts",
+        "answer_column": f"{pred_col}.answer",
+    }
+    ```
+
+    For more complex situations, you can use a function to extract the data:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "contexts_column": lambda row: [row[pred_col]["context_message"]],
+        "answer_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
+    }
+    ```
+    """
+    warnings.filterwarnings(
+        "ignore",
+        category=FutureWarning,
+        message="promote has been superseded by promote_options='default'.",
+    )
+
+    required_columns = {
+        "answer": answer_column,
+        "contexts": contexts_column,
+    }
+
+    df = get_renamed_columns(dataset.df, required_columns)
+
+    result_df = evaluate(
+        Dataset.from_pandas(df),
+        metrics=[faithfulness],
+    ).to_pandas()
+
+    fig_histogram = px.histogram(x=result_df["faithfulness"].to_list(), nbins=10)
+    fig_box = px.box(x=result_df["faithfulness"].to_list())
+
+    return (
+        {
+            "Scores": result_df[["contexts", "answer", "faithfulness"]],
+            "Aggregate Scores": [
+                {
+                    "Mean Score": result_df["faithfulness"].mean(),
+                    "Median Score": result_df["faithfulness"].median(),
+                    "Max Score": result_df["faithfulness"].max(),
+                    "Min Score": result_df["faithfulness"].min(),
+                    "Standard Deviation": result_df["faithfulness"].std(),
+                    "Count": len(result_df),
+                }
+            ],
+        },
+        fig_histogram,
+        fig_box,
+    )

validmind/tests/model_validation/ragas/utils.py

@@ -0,0 +1,66 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+
+def _udf_get_sub_col(x, root_col, sub_col):
+    if not isinstance(x, dict):
+        raise TypeError(f"Expected a dictionary in column '{root_col}', got {type(x)}.")
+
+    if sub_col not in x:
+        raise KeyError(
+            f"Sub-column '{sub_col}' not found in dictionary in column '{root_col}'."
+        )
+
+    return x[sub_col]
+
+
+def get_renamed_columns(df, column_map):
+    """Get a new df with columns renamed according to the column_map
+
+    Supports sub-column notation for getting values out of dictionaries that may be
+    stored in a column. Also supports
+
+    Args:
+        df (pd.DataFrame): The DataFrame to rename columns in.
+        column_map (dict): A dictionary mapping where the keys are the new column names
+        that ragas expects and the values are one of the following:
+            - The column name in the input dataframe
+            - A string in the format "root_col.sub_col" to get a sub-column from a dictionary
+            stored in a column.
+            - A function that takes the value of the column and returns the value to be
+            stored in the new column.
+
+    Returns:
+        pd.DataFrame: The DataFrame with columns renamed.
+    """
+    new_df = df.copy()
+
+    for new_name, source in column_map.items():
+        if callable(source):
+            try:
+                new_df[new_name] = new_df.apply(source, axis=1)
+            except Exception as e:
+                raise ValueError(
+                    f"Failed to apply function to DataFrame. Error: {str(e)}"
+                )
+
+        elif "." in source:
+            root_col, sub_col = source.split(".")
+
+            if root_col in new_df.columns:
+                new_df[new_name] = new_df[root_col].apply(
+                    lambda x: _udf_get_sub_col(x, root_col, sub_col)
+                )
+
+            else:
+                raise KeyError(f"Column '{root_col}' not found in DataFrame.")
+
+        else:
+            if source in new_df.columns:
+                new_df[new_name] = new_df[source]
+
+            else:
+                raise KeyError(f"Column '{source}' not found in DataFrame.")
+
+    return new_df

validmind/tests/model_validation/sklearn/OverfitDiagnosis.py

@@ -90,7 +90,7 @@ class OverfitDiagnosis(ThresholdTest):
             raise ValueError("features_columns must be provided in params")
 
         if self.params["features_columns"] is None:
-            features_list = self.inputs.datasets[0].get_features_columns()
+            features_list = self.inputs.datasets[0].feature_columns
         else:
             features_list = self.params["features_columns"]
 
@@ -101,8 +101,7 @@ class OverfitDiagnosis(ThresholdTest):
 
         # Check if all elements from features_list are present in the feature columns
         all_present = all(
-            elem in self.inputs.datasets[0].get_features_columns()
-            for elem in features_list
+            elem in self.inputs.datasets[0].feature_columns for elem in features_list
         )
         if not all_present:
             raise ValueError(
@@ -134,10 +133,7 @@ class OverfitDiagnosis(ThresholdTest):
 
         for feature_column in features_list:
             bins = 10
-            if (
-                feature_column
-                in self.inputs.datasets[0].get_categorical_features_columns()
-            ):
+            if feature_column in self.inputs.datasets[0].feature_columns_categorical:
                 bins = len(train_df[feature_column].unique())
             train_df["bin"] = pd.cut(train_df[feature_column], bins=bins)
 

validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py

@@ -71,15 +71,14 @@ class PermutationFeatureImportance(Metric):
         x = self.inputs.dataset.x_df()
         y = self.inputs.dataset.y_df()
 
-        model_library = self.inputs.model.model_library()
-        if (
-            model_library == "statsmodels"
-            or model_library == "pytorch"
-            or model_library == "catboost"
-            or model_library == "transformers"
-            or model_library == "R"
-        ):
-            raise SkipTestError(f"Skipping PFI for {model_library} models")
+        if self.inputs.model.library in [
+            "statsmodels",
+            "pytorch",
+            "catboost",
+            "transformers",
+            "R",
+        ]:
+            raise SkipTestError(f"Skipping PFI for {self.inputs.model.library} models")
 
         pfi_values = permutation_importance(
             self.inputs.model.model,

validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py

@@ -92,9 +92,9 @@ class PopulationStabilityIndex(Metric):
         # The data looks like this: [{"initial": 2652, "percent_initial": 0.5525, "new": 830, "percent_new": 0.5188, "psi": 0.0021},...
         psi_table = [
             {
-                "Bin": i
-                if i < (len(metric_value) - 1)
-                else "Total",  # The last bin is the "Total" bin
+                "Bin": (
+                    i if i < (len(metric_value) - 1) else "Total"
+                ),  # The last bin is the "Total" bin
                 "Count Initial": values["initial"],
                 "Percent Initial (%)": values["percent_initial"] * 100,
                 "Count New": values["new"],
@@ -180,13 +180,8 @@ class PopulationStabilityIndex(Metric):
         return psi_df.to_dict(orient="records")
 
     def run(self):
-        model_library = self.inputs.model.model_library()
-        if (
-            model_library == "statsmodels"
-            or model_library == "pytorch"
-            or model_library == "catboost"
-        ):
-            logger.info(f"Skiping PSI for {model_library} models")
+        if self.inputs.model.library in ["statsmodels", "pytorch", "catboost"]:
+            logger.info(f"Skiping PSI for {self.inputs.model.library} models")
             return
 
         num_bins = self.params["num_bins"]

validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py

@@ -9,6 +9,7 @@ import plotly.graph_objects as go
 from sklearn.metrics import precision_recall_curve
 
 from validmind.errors import SkipTestError
+from validmind.models import FoundationModel
 from validmind.vm_models import Figure, Metric
 
 
@@ -42,7 +43,7 @@ class PrecisionRecallCurve(Metric):
     different threshold levels.
 
     **Limitations**:
-    * This metric is only applicable to binary classification models – it raises errors for multiclass classification
+    * This metric is only applicable to binary classification models - it raises errors for multiclass classification
     models or Foundation models.
     * It may not fully represent the overall accuracy of the model if the cost of false positives and false negatives
     are extremely different, or if the dataset is heavily imbalanced.
@@ -62,7 +63,7 @@ class PrecisionRecallCurve(Metric):
     }
 
     def run(self):
-        if self.inputs.model.model_library() == "FoundationModel":
+        if isinstance(self.inputs.model, FoundationModel):
             raise SkipTestError("Skipping PrecisionRecallCurve for Foundation models")
 
         y_true = self.inputs.dataset.y

validmind/tests/model_validation/sklearn/ROCCurve.py

@@ -9,6 +9,7 @@ import plotly.graph_objects as go
 from sklearn.metrics import roc_auc_score, roc_curve
 
 from validmind.errors import SkipTestError
+from validmind.models import FoundationModel
 from validmind.vm_models import Figure, Metric
 
 
@@ -70,7 +71,7 @@ class ROCCurve(Metric):
     }
 
     def run(self):
-        if self.inputs.model.model_library() == "FoundationModel":
+        if isinstance(self.inputs.model, FoundationModel):
             raise SkipTestError("Skipping ROCCurve for Foundation models")
 
         y_true = self.inputs.dataset.y

validmind/tests/model_validation/sklearn/RegressionR2Square.py

@@ -90,7 +90,7 @@ class RegressionR2Square(Metric):
             }
         )
 
-        X_columns = self.inputs.datasets[0].get_features_columns()
+        X_columns = self.inputs.datasets[0].feature_columns
         adj_r2_train = adj_r2_score(
             y_train_true, y_train_pred, len(y_train_true), len(X_columns)
         )

validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py

@@ -109,12 +109,11 @@ class RobustnessDiagnosis(ThresholdTest):
 
         features_list = self.params["features_columns"]
         if features_list is None:
-            features_list = self.inputs.datasets[0].get_numeric_features_columns()
+            features_list = self.inputs.datasets[0].feature_columns
 
         # Check if all elements from features_list are present in the numerical feature columns
         all_present = all(
-            elem in self.inputs.datasets[0].get_numeric_features_columns()
-            for elem in features_list
+            elem in self.inputs.datasets[0].feature_columns for elem in features_list
         )
         if not all_present:
             raise ValueError(

validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py

@@ -11,6 +11,7 @@ import shap
 
 from validmind.errors import UnsupportedModelForSHAPError
 from validmind.logging import get_logger
+from validmind.models import CatBoostModel, SKlearnModel, StatsModelsModel
 from validmind.vm_models import Figure, Metric
 
 logger = get_logger(__name__)
@@ -131,20 +132,14 @@ class SHAPGlobalImportance(Metric):
         )
 
     def run(self):
-        model_library = self.inputs.model.model_library()
-        if model_library in [
-            "statsmodels",
-            "pytorch",
-            "catboost",
-            "transformers",
-            "FoundationModel",
-            "R",
-        ]:
-            logger.info(f"Skiping SHAP for {model_library} models")
+        if not isinstance(self.inputs.model, SKlearnModel) or isinstance(
+            self.inputs.model, (CatBoostModel, StatsModelsModel)
+        ):
+            logger.info(f"Skiping SHAP for {self.inputs.model.library} models")
             return
 
         trained_model = self.inputs.model.model
-        model_class = self.inputs.model.model_class()
+        model_class = self.inputs.model.class_
 
         # the shap library generates a bunch of annoying warnings that we don't care about
         warnings.filterwarnings("ignore", category=UserWarning)
@@ -176,6 +171,7 @@ class SHAPGlobalImportance(Metric):
                 ),
             )
         else:
+            model_class = "<ExternalModel>" if model_class is None else model_class
             raise UnsupportedModelForSHAPError(
                 f"Model {model_class} not supported for SHAP importance."
             )

validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py

@@ -113,7 +113,7 @@ class WeakspotsDiagnosis(ThresholdTest):
                 raise ValueError(f"Threshold for metric {metric} is missing")
 
         if self.params["features_columns"] is None:
-            features_list = self.inputs.datasets[0].get_features_columns()
+            features_list = self.inputs.datasets[0].feature_columns
         else:
             features_list = self.params["features_columns"]
 
@@ -124,8 +124,7 @@ class WeakspotsDiagnosis(ThresholdTest):
 
         # Check if all elements from features_list are present in the feature columns
         all_present = all(
-            elem in self.inputs.datasets[0].get_features_columns()
-            for elem in features_list
+            elem in self.inputs.datasets[0].feature_columns for elem in features_list
         )
         if not all_present:
             raise ValueError(
@@ -150,7 +149,7 @@ class WeakspotsDiagnosis(ThresholdTest):
         results_headers.extend(self.default_metrics.keys())
         for feature in features_list:
             bins = 10
-            if feature in self.inputs.datasets[0].get_categorical_features_columns():
+            if feature in self.inputs.datasets[0].feature_columns_categorical:
                 bins = len(train_df[feature].unique())
             train_df["bin"] = pd.cut(train_df[feature], bins=bins)
 

validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py

@@ -89,7 +89,7 @@ class RegressionModelForecastPlot(Metric):
         figures = []
 
         for i, fitted_model in enumerate(model_list):
-            feature_columns = datasets[0].get_features_columns()
+            feature_columns = datasets[0].feature_columns
 
             train_ds = datasets[0]
             test_ds = datasets[1]

validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py

@@ -98,7 +98,7 @@ class RegressionModelForecastPlotLevels(Metric):
         figures = []
 
         for i, fitted_model in enumerate(model_list):
-            feature_columns = datasets[0].get_features_columns()
+            feature_columns = datasets[0].feature_columns
 
             train_ds = datasets[0]
             test_ds = datasets[1]