validmind 2.5.25__py3-none-any.whl → 2.6.8__py3-none-any.whl

validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -14,22 +14,25 @@ from .utils import get_ragas_config, get_renamed_columns
 
 try:
     from ragas import evaluate
-    from ragas.metrics import context_entity_recall
+    from ragas.metrics import ContextEntityRecall as context_entity_recall
 except ImportError as e:
-    raise MissingDependencyError(
-        "Missing required package `ragas` for ContextEntityRecall. "
-        "Please run `pip install validmind[llm]` to use LLM tests",
-        required_dependencies=["ragas"],
-        extra="llm",
-    ) from e
+    if "ragas" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `ragas` for ContextEntityRecall. "
+            "Please run `pip install validmind[llm]` to use LLM tests",
+            required_dependencies=["ragas"],
+            extra="llm",
+        ) from e
+
+    raise e
 
 
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
 def ContextEntityRecall(
     dataset,
-    contexts_column: str = "contexts",
-    ground_truth_column: str = "ground_truth",
+    retrieved_contexts_column: str = "retrieved_contexts",
+    reference_column: str = "reference",
 ):
     """
     Evaluates the context entity recall for dataset entries and visualizes the results.
@@ -37,18 +40,18 @@ def ContextEntityRecall(
     ### Overview
 
     This metric gives the measure of recall of the retrieved context, based on the
-    number of entities present in both `ground_truths` and `contexts` relative to the
-    number of entities present in the `ground_truths` alone. Simply put, it is a measure
-    of what fraction of entities are recalled from `ground_truths`. This metric is
+    number of entities present in both `reference` and `retrieved_contexts` relative to the
+    number of entities present in the `reference` alone. Simply put, it is a measure
+    of what fraction of entities are recalled from `reference`. This metric is
     useful in fact-based use cases like tourism help desk, historical QA, etc. This
     metric can help evaluate the retrieval mechanism for entities, based on comparison
-    with entities present in `ground_truths`, because in cases where entities matter,
-    we need the `contexts` which cover them.
+    with entities present in `reference`, because in cases where entities matter,
+    we need the `retrieved_contexts` which cover them.
 
     ### Formula
 
     To compute this metric, we use two sets, $GE$ and $CE$, representing the set of
-    entities present in `ground_truths` and set of entities present in `contexts`
+    entities present in `reference` and set of entities present in `retrieved_contexts`
     respectively. We then take the number of elements in intersection of these sets and
     divide it by the number of elements present in the $GE$, given by the formula:
 
@@ -60,20 +63,20 @@ def ContextEntityRecall(
 
     This metric requires the following columns in your dataset:
 
-    - `contexts` (List[str]): A list of text contexts which will be evaluated to make
-    sure if they contain the entities present in the ground truth.
-    - `ground_truth` (str): The ground truth text from which the entities will be
-    extracted and compared with the entities in the `contexts`.
+    - `retrieved_contexts` (List[str]): A list of text contexts which will be evaluated to make
+    sure if they contain the entities present in the `reference`.
+    - `reference` (str): The ground truth text from which the entities will be
+    extracted and compared with the entities in the `retrieved_contexts`.
 
     If the above data is not in the appropriate column, you can specify different column
-    names for these fields using the parameters `contexts_column`, and `ground_truth_column`.
+    names for these fields using the parameters `retrieved_contexts_column`, and `reference_column`.
 
     For example, if your dataset has this data stored in different columns, you can
     pass the following parameters:
     ```python
     {
-        "contexts_column": "context_info"
-        "ground_truth_column": "my_ground_truth_col",
+        "retrieved_contexts_column": "context_info",
+        "reference_column": "my_ground_truth_col",
     }
     ```
 
@@ -82,8 +85,8 @@ def ContextEntityRecall(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "contexts_column": f"{pred_col}.contexts",
-        "ground_truth_column": "my_ground_truth_col",
+        "retrieved_contexts_column": f"{pred_col}.contexts",
+        "reference_column": "my_ground_truth_col",
     }
     ```
 
@@ -91,8 +94,8 @@ def ContextEntityRecall(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "contexts_column": lambda row: [row[pred_col]["context_message"]],
-        "ground_truth_column": "my_ground_truth_col",
+        "retrieved_contexts_column": lambda row: [row[pred_col]["context_message"]],
+        "reference_column": "my_ground_truth_col",
     }
     ```
     """
@@ -103,37 +106,37 @@ def ContextEntityRecall(
     )
 
     required_columns = {
-        "ground_truth": ground_truth_column,
-        "contexts": contexts_column,
+        "reference": reference_column,
+        "retrieved_contexts": retrieved_contexts_column,
     }
 
     df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[context_entity_recall], **get_ragas_config()
+        Dataset.from_pandas(df), metrics=[context_entity_recall()], **get_ragas_config()
     ).to_pandas()
 
-    fig_histogram = px.histogram(
-        x=result_df["context_entity_recall"].to_list(), nbins=10
-    )
-    fig_box = px.box(x=result_df["context_entity_recall"].to_list())
+    score_column = "context_entity_recall"
+
+    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
+    fig_box = px.box(x=result_df[score_column].to_list())
 
     return (
         {
-            # "Scores (will not be uploaded to UI)": result_df[
+            # "Scores (will not be uploaded to ValidMind Platform)": result_df[
             #     [
-            #         "contexts",
-            #         "ground_truth",
+            #         "retrieved_contexts",
+            #         "reference",
             #         "context_entity_recall",
             #     ]
             # ],
             "Aggregate Scores": [
                 {
-                    "Mean Score": result_df["context_entity_recall"].mean(),
-                    "Median Score": result_df["context_entity_recall"].median(),
-                    "Max Score": result_df["context_entity_recall"].max(),
-                    "Min Score": result_df["context_entity_recall"].min(),
-                    "Standard Deviation": result_df["context_entity_recall"].std(),
+                    "Mean Score": result_df[score_column].mean(),
+                    "Median Score": result_df[score_column].median(),
+                    "Max Score": result_df[score_column].max(),
+                    "Min Score": result_df[score_column].min(),
+                    "Standard Deviation": result_df[score_column].std(),
                     "Count": result_df.shape[0],
                 }
             ],

validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -14,23 +14,26 @@ from .utils import get_ragas_config, get_renamed_columns
 
 try:
     from ragas import evaluate
-    from ragas.metrics import context_precision
+    from ragas.metrics import LLMContextPrecisionWithReference as context_precision
 except ImportError as e:
-    raise MissingDependencyError(
-        "Missing required package `ragas` for ContextPrecision. "
-        "Please run `pip install validmind[llm]` to use LLM tests",
-        required_dependencies=["ragas"],
-        extra="llm",
-    ) from e
+    if "ragas" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `ragas` for ContextPrecision. "
+            "Please run `pip install validmind[llm]` to use LLM tests",
+            required_dependencies=["ragas"],
+            extra="llm",
+        ) from e
+
+    raise e
 
 
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
 def ContextPrecision(
     dataset,
-    question_column: str = "question",
-    contexts_column: str = "contexts",
-    ground_truth_column: str = "ground_truth",
+    user_input_column: str = "user_input",
+    retrieved_contexts_column: str = "retrieved_contexts",
+    reference_column: str = "reference",
 ):  # noqa: B950
     """
     Context Precision is a metric that evaluates whether all of the ground-truth
@@ -53,22 +56,22 @@ def ContextPrecision(
 
     This metric requires the following columns in your dataset:
 
-    - `question` (str): The text query that was input into the model.
-    - `contexts` (List[str]): A list of text contexts which are retrieved and which
+    - `user_input` (str): The text query that was input into the model.
+    - `retrieved_contexts` (List[str]): A list of text contexts which are retrieved and which
     will be evaluated to make sure they contain relevant info in the correct order.
-    - `ground_truth` (str): The ground truth text to compare with the retrieved contexts.
+    - `reference` (str): The ground truth text to compare with the retrieved contexts.
 
     If the above data is not in the appropriate column, you can specify different column
-    names for these fields using the parameters `question_column`, `contexts_column`
-    and `ground_truth_column`.
+    names for these fields using the parameters `user_input_column`, `retrieved_contexts_column`
+    and `reference_column`.
 
     For example, if your dataset has this data stored in different columns, you can
     pass the following parameters:
     ```python
     {
-        "question_column": "question",
-        "contexts_column": "context_info"
-        "ground_truth_column": "my_ground_truth_col",
+        "user_input_column": "question",
+        "retrieved_contexts_column": "context_info",
+        "reference_column": "my_ground_truth_col",
     }
     ```
 
@@ -77,8 +80,8 @@ def ContextPrecision(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "contexts_column": f"{pred_col}.contexts",
-        "ground_truth_column": "my_ground_truth_col",
+        "retrieved_contexts_column": f"{pred_col}.retrieved_contexts",
+        "reference_column": "my_ground_truth_col",
     }
     ```
 
@@ -86,8 +89,8 @@ def ContextPrecision(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "contexts_column": lambda x: [x[pred_col]["context_message"]],
-        "ground_truth_column": "my_ground_truth_col",
+        "retrieved_contexts_column": lambda x: [x[pred_col]["context_message"]],
+        "reference_column": "my_ground_truth_col",
     }
     ```
     """
@@ -98,32 +101,34 @@ def ContextPrecision(
     )
 
     required_columns = {
-        "question": question_column,
-        "contexts": contexts_column,
-        "ground_truth": ground_truth_column,
+        "user_input": user_input_column,
+        "retrieved_contexts": retrieved_contexts_column,
+        "reference": reference_column,
     }
 
     df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[context_precision], **get_ragas_config()
+        Dataset.from_pandas(df), metrics=[context_precision()], **get_ragas_config()
     ).to_pandas()
 
-    fig_histogram = px.histogram(x=result_df["context_precision"].to_list(), nbins=10)
-    fig_box = px.box(x=result_df["context_precision"].to_list())
+    score_column = "llm_context_precision_with_reference"
+
+    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
+    fig_box = px.box(x=result_df[score_column].to_list())
 
     return (
         {
-            # "Scores (will not be uploaded to UI)": result_df[
-            #     ["question", "contexts", "ground_truth", "context_precision"]
+            # "Scores (will not be uploaded to ValidMind Platform)": result_df[
+            #     ["user_input", "retrieved_contexts", "reference", "llm_context_precision_with_reference"]
             # ],
             "Aggregate Scores": [
                 {
-                    "Mean Score": result_df["context_precision"].mean(),
-                    "Median Score": result_df["context_precision"].median(),
-                    "Max Score": result_df["context_precision"].max(),
-                    "Min Score": result_df["context_precision"].min(),
-                    "Standard Deviation": result_df["context_precision"].std(),
+                    "Mean Score": result_df[score_column].mean(),
+                    "Median Score": result_df[score_column].median(),
+                    "Max Score": result_df[score_column].max(),
+                    "Min Score": result_df[score_column].min(),
+                    "Standard Deviation": result_df[score_column].std(),
                     "Count": result_df.shape[0],
                 }
             ],

validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py

@@ -0,0 +1,133 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import warnings
+
+import plotly.express as px
+from datasets import Dataset
+
+from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
+
+from .utils import get_ragas_config, get_renamed_columns
+
+try:
+    from ragas import evaluate
+    from ragas.metrics import LLMContextPrecisionWithoutReference as context_precision
+except ImportError as e:
+    if "ragas" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `ragas` for ContextPrecision. "
+            "Please run `pip install validmind[llm]` to use LLM tests",
+            required_dependencies=["ragas"],
+            extra="llm",
+        ) from e
+
+    raise e
+
+
+@tags("ragas", "llm", "retrieval_performance")
+@tasks("text_qa", "text_generation", "text_summarization", "text_classification")
+def ContextPrecisionWithoutReference(
+    dataset,
+    user_input_column: str = "user_input",
+    retrieved_contexts_column: str = "retrieved_contexts",
+    response_column: str = "response",
+):  # noqa: B950
+    """
+    Context Precision Without Reference is a metric used to evaluate the relevance of
+    retrieved contexts compared to the expected response for a given user input. This
+    metric compares each retrieved context (or chunk) with the response to estimate
+    if the retrieved context is relevant.
+
+    This metric can be used when you have both retrieved contexts and associated
+    reference contexts for a `user_input`. Using a Language Model (LLM), it determines
+    the relevance of each retrieved context by comparing it directly with the response,
+    producing scores between 0 and 1, where higher scores indicate better precision in
+    retrieving relevant contexts.
+
+    ### Configuring Columns
+
+    This metric requires the following columns in your dataset:
+
+    - `user_input` (str): The user query or input to the model.
+    - `retrieved_contexts` (List[str]): A list of text contexts retrieved for the
+      user input that will be evaluated for relevance.
+    - `response` (str): The model’s output response associated with the user input.
+
+    If your dataset stores this data in different columns, you can specify alternate
+    column names using the parameters `user_input_column`, `retrieved_contexts_column`,
+    and `response_column`.
+
+    Example configuration for custom column names:
+    ```python
+    {
+        "user_input_column": "user_query",
+        "retrieved_contexts_column": "retrieved_texts",
+        "response_column": "model_output",
+    }
+    ```
+
+    For datasets with data stored as dictionaries in other columns, specify the
+    column and key like so:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "retrieved_contexts_column": f"{pred_col}.contexts",
+        "response_column": f"{pred_col}.response",
+    }
+    ```
+
+    Alternatively, for complex situations, you may use a function to extract data:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "retrieved_contexts_column": lambda x: [x[pred_col]["context_message"]],
+        "response_column": "my_response_col",
+    }
+    ```
+    """
+
+    warnings.filterwarnings(
+        "ignore",
+        category=FutureWarning,
+        message="promote has been superseded by promote_options='default'.",
+    )
+
+    required_columns = {
+        "user_input": user_input_column,
+        "retrieved_contexts": retrieved_contexts_column,
+        "response": response_column,
+    }
+
+    df = get_renamed_columns(dataset._df, required_columns)
+
+    result_df = evaluate(
+        Dataset.from_pandas(df), metrics=[context_precision()], **get_ragas_config()
+    ).to_pandas()
+
+    score_column = "llm_context_precision_without_reference"
+
+    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
+    fig_box = px.box(x=result_df[score_column].to_list())
+
+    return (
+        {
+            # "Scores (will not be uploaded to ValidMind Platform)": result_df[
+            #     ["user_input", "retrieved_contexts", "response", "llm_context_precision_without_reference"]
+            # ],
+            "Aggregate Scores": [
+                {
+                    "Mean Score": result_df[score_column].mean(),
+                    "Median Score": result_df[score_column].median(),
+                    "Max Score": result_df[score_column].max(),
+                    "Min Score": result_df[score_column].min(),
+                    "Standard Deviation": result_df[score_column].std(),
+                    "Count": result_df.shape[0],
+                }
+            ],
+        },
+        fig_histogram,
+        fig_box,
+    )

validmind/tests/model_validation/ragas/ContextRecall.py

@@ -14,23 +14,26 @@ from .utils import get_ragas_config, get_renamed_columns
 
 try:
     from ragas import evaluate
-    from ragas.metrics import context_recall
+    from ragas.metrics import LLMContextRecall as context_recall
 except ImportError as e:
-    raise MissingDependencyError(
-        "Missing required package `ragas` for ContextRecall. "
-        "Please run `pip install validmind[llm]` to use LLM tests",
-        required_dependencies=["ragas"],
-        extra="llm",
-    ) from e
+    if "ragas" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `ragas` for ContextRecall. "
+            "Please run `pip install validmind[llm]` to use LLM tests",
+            required_dependencies=["ragas"],
+            extra="llm",
+        ) from e
+
+    raise e
 
 
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
 def ContextRecall(
     dataset,
-    question_column: str = "question",
-    contexts_column: str = "contexts",
-    ground_truth_column: str = "ground_truth",
+    user_input_column: str = "user_input",
+    retrieved_contexts_column: str = "retrieved_contexts",
+    reference_column: str = "reference",
 ):
     """
     Context recall measures the extent to which the retrieved context aligns with the
@@ -53,22 +56,22 @@ def ContextRecall(
 
     This metric requires the following columns in your dataset:
 
-    - `question` (str): The text query that was input into the model.
-    - `contexts` (List[str]): A list of text contexts which are retrieved and which
-    will be evaluated to make sure they contain all items in the ground truth.
-    - `ground_truth` (str): The ground truth text to compare with the retrieved contexts.
+    - `user_input` (str): The text query that was input into the model.
+    - `retrieved_contexts` (List[str]): A list of text contexts which are retrieved and
+    which will be evaluated to make sure they contain all items in the ground truth.
+    - `reference` (str): The ground truth text to compare with the retrieved contexts.
 
     If the above data is not in the appropriate column, you can specify different column
-    names for these fields using the parameters `question_column`, `contexts_column`
-    and `ground_truth_column`.
+    names for these fields using the parameters `user_input_column`,
+    `retrieved_contexts_column` and `reference_column`.
 
     For example, if your dataset has this data stored in different columns, you can
     pass the following parameters:
     ```python
     {
-        "question_column": "question",
-        "contexts_column": "context_info"
-        "ground_truth_column": "my_ground_truth_col",
+        "user_input_column": "user_input",
+        "retrieved_contexts_column": "retrieved_contexts",
+        "reference_column": "reference",
     }
     ```
 
@@ -77,8 +80,8 @@ def ContextRecall(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "contexts_column": f"{pred_col}.contexts",
-        "ground_truth_column": "my_ground_truth_col",
+        "retrieved_contexts_column": f"{pred_col}.retrieved_contexts",
+        "reference_column": f"{pred_col}.reference",
     }
     ```
 
@@ -86,8 +89,8 @@ def ContextRecall(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "contexts_column": lambda x: [x[pred_col]["context_message"]],
-        "ground_truth_column": "my_ground_truth_col",
+        "retrieved_contexts_column": lambda x: [x[pred_col]["retrieved_contexts"]],
+        "reference_column": "my_ground_truth_col",
     }
     ```
     """
@@ -98,32 +101,34 @@ def ContextRecall(
     )
 
     required_columns = {
-        "question": question_column,
-        "contexts": contexts_column,
-        "ground_truth": ground_truth_column,
+        "user_input": user_input_column,
+        "retrieved_contexts": retrieved_contexts_column,
+        "reference": reference_column,
     }
 
     df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[context_recall], **get_ragas_config()
+        Dataset.from_pandas(df), metrics=[context_recall()], **get_ragas_config()
     ).to_pandas()
 
-    fig_histogram = px.histogram(x=result_df["context_recall"].to_list(), nbins=10)
-    fig_box = px.box(x=result_df["context_recall"].to_list())
+    score_column = "context_recall"
+
+    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
+    fig_box = px.box(x=result_df[score_column].to_list())
 
     return (
         {
-            # "Scores (will not be uploaded to UI)": result_df[
+            # "Scores (will not be uploaded to ValidMind Platform)": result_df[
             #     ["question", "contexts", "ground_truth", "context_recall"]
             # ],
             "Aggregate Scores": [
                 {
-                    "Mean Score": result_df["context_recall"].mean(),
-                    "Median Score": result_df["context_recall"].median(),
-                    "Max Score": result_df["context_recall"].max(),
-                    "Min Score": result_df["context_recall"].min(),
-                    "Standard Deviation": result_df["context_recall"].std(),
+                    "Mean Score": result_df[score_column].mean(),
+                    "Median Score": result_df[score_column].median(),
+                    "Max Score": result_df[score_column].max(),
+                    "Min Score": result_df[score_column].min(),
+                    "Standard Deviation": result_df[score_column].std(),
                     "Count": result_df.shape[0],
                 }
             ],