validmind 2.1.0__py3-none-any.whl → 2.2.2__py3-none-any.whl

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -0,0 +1,131 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import warnings
+
+import plotly.express as px
+from datasets import Dataset
+from ragas import evaluate
+from ragas.metrics import answer_correctness
+
+from validmind import tags, tasks
+
+from .utils import get_renamed_columns
+
+
+@tags("ragas", "llm")
+@tasks("text_qa", "text_generation", "text_summarization")
+def AnswerCorrectness(
+    dataset,
+    question_column="question",
+    answer_column="answer",
+    ground_truth_column="ground_truth",
+):
+    """
+    Evaluates the correctness of answers in a dataset with respect to the provided ground
+    truths and visualizes the results in a histogram.
+
+    The assessment of Answer Correctness involves gauging the accuracy of the generated
+    answer when compared to the ground truth. This evaluation relies on the `ground truth`
+    and the `answer`, with scores ranging from 0 to 1. A higher score indicates a closer
+    alignment between the generated answer and the ground truth, signifying better
+    correctness.
+
+    Answer correctness encompasses two critical aspects: semantic similarity between the
+    generated answer and the ground truth, as well as factual similarity. These aspects
+    are combined using a weighted scheme to formulate the answer correctness score. Users
+    also have the option to employ a `threshold` value to round the resulting score to
+    a binary value (0 or 1) based on the threshold.
+
+    Factual correctness quantifies the factual overlap between the generated answer and
+    the ground truth answer. This is done using the concepts of:
+
+    - TP (True Positive): Facts or statements that are present in both the ground truth
+      and the generated answer.
+    - FP (False Positive): Facts or statements that are present in the generated answer
+      but not in the ground truth.
+    - FN (False Negative): Facts or statements that are present in the ground truth but
+      not in the generated answer.
+
+    ### Configuring Columns
+
+    This metric requires specific columns to be present in the dataset:
+    - `question` (str): The text prompt or query that was input into the model.
+    - `answer` (str): The text response generated by the model.
+    - `ground_truth` (str): The ground truth answer that the generated answer is compared
+    against.
+
+    If the above data is not in the appropriate column, you can specify different column
+    names for these fields using the parameters `question_column`, `answer_column`, and
+    `ground_truth_column`.
+
+    For example, if your dataset has this data stored in different columns, you can
+    pass the following parameters:
+    ```python
+    params = {
+        "question_column": "input_text",
+        "answer_column": "output_text",
+        "ground_truth_column": "human_answer",
+    }
+    ```
+
+    If answer and contexts are stored as a dictionary in another column, specify the
+    column and key like this:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "answer_column": f"{pred_col}.generated_answer",
+        "ground_truth_column": f"{pred_col}.contexts",
+    }
+    ```
+
+    For more complex data structures, you can use a function to extract the answers:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "answer_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
+        "ground_truth_column": lambda row: [row[pred_col]["context_message"]],
+    }
+    ```
+    """
+    warnings.filterwarnings(
+        "ignore",
+        category=FutureWarning,
+        message="promote has been superseded by promote_options='default'.",
+    )
+
+    required_columns = {
+        "question": question_column,
+        "answer": answer_column,
+        "ground_truth": ground_truth_column,
+    }
+
+    df = get_renamed_columns(dataset.df, required_columns)
+
+    result_df = evaluate(
+        Dataset.from_pandas(df), metrics=[answer_correctness]
+    ).to_pandas()
+
+    fig_histogram = px.histogram(x=result_df["answer_correctness"].to_list(), nbins=10)
+    fig_box = px.box(x=result_df["answer_correctness"].to_list())
+
+    return (
+        {
+            "Scores": result_df[
+                ["question", "answer", "ground_truth", "answer_correctness"]
+            ],
+            "Aggregate Scores": [
+                {
+                    "Mean Score": result_df["answer_correctness"].mean(),
+                    "Median Score": result_df["answer_correctness"].median(),
+                    "Max Score": result_df["answer_correctness"].max(),
+                    "Min Score": result_df["answer_correctness"].min(),
+                    "Standard Deviation": result_df["answer_correctness"].std(),
+                    "Count": len(result_df),
+                }
+            ],
+        },
+        fig_histogram,
+        fig_box,
+    )

validmind/tests/model_validation/ragas/AnswerRelevance.py

@@ -0,0 +1,134 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import warnings
+
+import plotly.express as px
+from datasets import Dataset
+from ragas import evaluate
+from ragas.metrics import answer_relevancy
+
+from validmind import tags, tasks
+
+from .utils import get_renamed_columns
+
+
+@tags("ragas", "llm", "rag_performance")
+@tasks("text_qa", "text_generation", "text_summarization")
+def AnswerRelevance(
+    dataset,
+    question_column="question",
+    contexts_column="contexts",
+    answer_column="answer",
+):
+    """
+    Assesses how pertinent the generated answer is to the given prompt.
+
+    The evaluation metric, Answer Relevancy, focuses on assessing how pertinent the
+    generated answer is to the given prompt. A lower score is assigned to answers that
+    are incomplete or contain redundant information and higher scores indicate better
+    relevancy. This metric is computed using the `question`, the `contexts` and the
+    `answer`.
+
+    The Answer Relevancy is defined as the mean cosine similartiy of the original
+    `question` to a number of artifical questions, which are generated (reverse-engineered)
+    based on the `answer`:
+
+    $$
+    \\text{answer relevancy} = \\frac{1}{N} \\sum_{i=1}^{N} cos(E_{g_i}, E_o)
+    $$
+    $$
+    \\text{answer relevancy} = \\frac{1}{N} \\sum_{i=1}^{N} \\frac{E_{g_i} \\cdot E_o}{\\|E_{g_i}\\|\\|E_o\\|}
+    $$
+
+    Where:
+    - $E_{g_i}$ is the embedding of the generated question $i$.
+    - $E_o$ is the embedding of the original question.
+    - $N$ is the number of generated questions - 3 by default.
+
+    **Note**: *This is a reference-free metric, meaning that it does not require a
+    `ground_truth` answer to compare against. A similar metric that does evaluate the
+    correctness of a generated answser with respect to a `ground_truth` answer is
+    `validmind.model_validation.ragas.AnswerCorrectness`.*
+
+    ### Configuring Columns
+
+    This metric requires the following columns in your dataset:
+    - `question` (str): The text query that was input into the model.
+    - `contexts` (List[str]): Any contextual information retrieved by the model before
+    generating an answer.
+    - `answer` (str): The response generated by the model.
+
+    If the above data is not in the appropriate column, you can specify different column
+    names for these fields using the parameters `question_column`, `answer_column`, and
+    `contexts_column`.
+
+    For example, if your dataset has this data stored in different columns, you can
+    pass the following parameters:
+    ```python
+    params = {
+        "question_column": "input_text",
+        "answer_column": "output_text",
+        "contexts_column": "context_info"
+    }
+    ```
+
+    If answer and contexts are stored as a dictionary in another column, specify the
+    column and key like this:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "answer_column": f"{pred_col}.generated_answer",
+        "contexts_column": f"{pred_col}.contexts",
+    }
+    ```
+
+    For more complex data structures, you can use a function to extract the answers:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "answer_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
+        "contexts_column": lambda row: [row[pred_col]["context_message"]],
+    }
+    ```
+    """
+    warnings.filterwarnings(
+        "ignore",
+        category=FutureWarning,
+        message="promote has been superseded by promote_options='default'.",
+    )
+
+    required_columns = {
+        "question": question_column,
+        "answer": answer_column,
+        "contexts": contexts_column,
+    }
+
+    df = get_renamed_columns(dataset.df, required_columns)
+
+    result_df = evaluate(
+        Dataset.from_pandas(df),
+        metrics=[answer_relevancy],
+    ).to_pandas()
+
+    fig_histogram = px.histogram(x=result_df["answer_relevancy"].to_list(), nbins=10)
+    fig_box = px.box(x=result_df["answer_relevancy"].to_list())
+
+    return (
+        {
+            "Scores": result_df[["question", "contexts", "answer", "answer_relevancy"]],
+            "Aggregate Scores": [
+                {
+                    "Mean Score": result_df["answer_relevancy"].mean(),
+                    "Median Score": result_df["answer_relevancy"].median(),
+                    "Max Score": result_df["answer_relevancy"].max(),
+                    "Min Score": result_df["answer_relevancy"].min(),
+                    "Standard Deviation": result_df["answer_relevancy"].std(),
+                    "Count": len(result_df),
+                }
+            ],
+        },
+        fig_histogram,
+        fig_box,
+    )

validmind/tests/model_validation/ragas/AnswerSimilarity.py

@@ -0,0 +1,119 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import warnings
+
+import plotly.express as px
+from datasets import Dataset
+from ragas import evaluate
+from ragas.metrics import answer_similarity
+
+from validmind import tags, tasks
+
+from .utils import get_renamed_columns
+
+
+@tags("ragas", "llm")
+@tasks("text_qa", "text_generation", "text_summarization")
+def AnswerSimilarity(
+    dataset,
+    answer_column="answer",
+    ground_truth_column="ground_truth",
+):
+    """
+    Calculates the semantic similarity between generated answers and ground truths
+
+    The concept of Answer Semantic Similarity pertains to the assessment of the semantic
+    resemblance between the generated answer and the ground truth. This evaluation is
+    based on the `ground_truth` and the `answer`, with values falling within the range
+    of 0 to 1. A higher score signifies a better alignment between the generated answer
+    and the ground truth.
+
+    Measuring the semantic similarity between answers can offer valuable insights into
+    the quality of the generated response. This evaluation utilizes a cross-encoder
+    model to calculate the semantic similarity score.
+
+    See this paper for more details: https://arxiv.org/pdf/2108.06130.pdf
+
+    The following steps are involved in computing the answer similarity score:
+    1. Vectorize the ground truth answer using the specified embedding model.
+    2. Vectorize the generated answer using the same embedding model.
+    3. Compute the cosine similarity between the two vectors.
+
+    ### Configuring Columns
+
+    This metric requires the following columns in your dataset:
+    - `answer` (str): The text response generated by the model.
+    - `ground_truth` (str): The ground truth answer that the generated answer is compared
+    against.
+
+    If the above data is not in the appropriate column, you can specify different column
+    names for these fields using the parameters `answer_column`, and `ground_truth_column`.
+
+    For example, if your dataset has this data stored in different columns, you can
+    pass the following parameters:
+    ```python
+    {
+        "answer_column": "llm_output_col",
+        "ground_truth_column": "my_ground_truth_col",
+    }
+    ```
+
+    If answer is stored as a dictionary in another column, specify the column and key
+    like this:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "answer_column": f"{pred_col}.generated_answer",
+        "ground_truth_column": "my_ground_truth_col",
+    }
+    ```
+
+    For more complex situations, you can use a function to extract the data:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "answer_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
+        "ground_truth_column": "my_ground_truth_col",
+    }
+    ```
+    """
+    warnings.filterwarnings(
+        "ignore",
+        category=FutureWarning,
+        message="promote has been superseded by promote_options='default'.",
+    )
+
+    required_columns = {
+        "answer": answer_column,
+        "ground_truth": ground_truth_column,
+    }
+
+    df = get_renamed_columns(dataset.df, required_columns)
+
+    result_df = evaluate(
+        Dataset.from_pandas(df),
+        metrics=[answer_similarity],
+    ).to_pandas()
+
+    fig_histogram = px.histogram(x=result_df["answer_similarity"].to_list(), nbins=10)
+    fig_box = px.box(x=result_df["answer_similarity"].to_list())
+
+    return (
+        {
+            "Scores": result_df[["answer", "ground_truth", "answer_similarity"]],
+            "Aggregate Scores": [
+                {
+                    "Mean Score": result_df["answer_similarity"].mean(),
+                    "Median Score": result_df["answer_similarity"].median(),
+                    "Max Score": result_df["answer_similarity"].max(),
+                    "Min Score": result_df["answer_similarity"].min(),
+                    "Standard Deviation": result_df["answer_similarity"].std(),
+                    "Count": len(result_df),
+                }
+            ],
+        },
+        fig_histogram,
+        fig_box,
+    )

validmind/tests/model_validation/ragas/AspectCritique.py

@@ -0,0 +1,167 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import warnings
+
+import plotly.express as px
+from datasets import Dataset
+from ragas import evaluate
+from ragas.metrics.critique import AspectCritique as _AspectCritique
+from ragas.metrics.critique import (
+    coherence,
+    conciseness,
+    correctness,
+    harmfulness,
+    maliciousness,
+)
+
+from validmind import tags, tasks
+
+from .utils import get_renamed_columns
+
+aspect_map = {
+    "coherence": coherence,
+    "conciseness": conciseness,
+    "correctness": correctness,
+    "harmfulness": harmfulness,
+    "maliciousness": maliciousness,
+}
+
+
+@tags("ragas", "llm", "qualitative")
+@tasks("text_summarization", "text_generation", "text_qa")
+def AspectCritique(
+    dataset,
+    question_column="question",
+    answer_column="answer",
+    contexts_column="contexts",
+    aspects: list = [
+        "coherence",
+        "conciseness",
+        "correctness",
+        "harmfulness",
+        "maliciousness",
+    ],
+    additional_aspects: list = [],
+):
+    """
+    Evaluates generations against the following aspects: harmfulness, maliciousness,
+    coherence, correctness, and conciseness.
+
+    ### Overview:
+
+    This is designed to assess submissions against predefined and user-defined "aspects".
+    For each aspect, a judge LLM is prompted to critique a piece of generated text based
+    on a description of the aspect. The output of this evaluation is a binary (0/1 = yes/no)
+    score that indicates whether the submission aligns with the defined aspect or not.
+
+    ### Inputs and Outputs:
+
+    The input to this metric is a dataset containing the input `question` (prompt to the LLM)
+    and the `answer` (text generated by the LLM). Any retrieved `contexts` can also be
+    included to enhance the evaluation.
+
+    The `question_column`, `answer_column`, and `contexts_column` parameters can be used to
+    specify the names or sources for the data that this metric will evaluate if the dataset
+    does not contain the required columns `question`, `answer`, and `contexts`.
+
+    By default, the aspects evaluated are harmfulness, maliciousness, coherence,
+    correctness, and conciseness. To change the aspects evaluated, the `aspects` parameter
+    can be set to a list containing any of these aspects.
+
+    To add custom aspects, the `additional_aspects` parameter can be passed as a list
+    of tuples where each tuple contains the aspect name and a description of the aspect
+    that the judge LLM will use to critique the submission.
+
+    The output of this metric is a table of scores for each aspect where the aspect score
+    is the number of "yes" scores divided by the total number of submissions:
+    $$
+    \\text{aspect score} = \\frac{\\text{number of "yes" scores}}{\\text{total number of submissions}}
+    $$
+
+    ### Examples:
+
+    - **Mapping to Required Columns:** If the dataset does not contain the columns required
+    to run this metric (i.e., `question`, `answer`, and `contexts`), the
+
+    ```python
+    pred_col = my_vm_dataset.prediction_column(my_vm_model)
+    run_test(
+        "validmind.model_validation.ragas.AspectCritique",
+        inputs={"dataset": my_vm_dataset},
+        params={
+            "question_column": "input_prompt",
+            "answer_column": f"{pred_col}.llm_output",
+            "contexts_column": lambda row: [row[pred_col]["context_message"]],
+        },
+    )
+    ```
+
+    - **Custom Aspects:** To evaluate custom aspects, the `additional_aspects` parameter can
+    be set to a list of tuples where each tuple contains the aspect name and a description
+    of the aspect that the judge LLM will use to critique the submission. For example, to
+    evaluate whether the LLM-generated text has a "professional tone", the `additional_aspects`
+    parameter can be set like this:
+
+    ```python
+    run_test(
+        "validmind.model_validation.ragas.AspectCritique",
+        inputs={"dataset": my_vm_dataset},
+        params={
+            "additional_aspects": [
+                ("professionalism", "Does the text have a professional tone?"),
+            ],
+        },
+    )
+    ```
+    """
+    warnings.filterwarnings(
+        "ignore",
+        category=FutureWarning,
+        message="promote has been superseded by promote_options='default'.",
+    )
+
+    required_columns = {
+        "question": question_column,
+        "answer": answer_column,
+        "contexts": contexts_column,
+    }
+
+    df = get_renamed_columns(dataset.df, required_columns)
+
+    built_in_aspects = [aspect_map[aspect] for aspect in aspects]
+    custom_aspects = [
+        _AspectCritique(name=name, definition=description)
+        for name, description in additional_aspects
+    ]
+    all_aspects = [*built_in_aspects, *custom_aspects]
+
+    result_df = evaluate(Dataset.from_pandas(df), metrics=all_aspects).to_pandas()
+
+    df_melted = result_df.melt(
+        id_vars=["question", "answer", "contexts"],
+        value_vars=[aspect.name for aspect in all_aspects],
+        var_name="Metric",
+        value_name="Result",
+    )
+    df_counts = df_melted.groupby(["Metric", "Result"]).size().reset_index(name="Count")
+    df_counts["Result"] = df_counts["Result"].map({0: "Fail", 1: "Pass"})
+
+    fig = px.bar(
+        df_counts,
+        x="Metric",
+        y="Count",
+        color="Result",
+        color_discrete_map={"Fail": "red", "Pass": "green"},
+        labels={"Count": "Pass vs Fail Count", "Metric": "Aspect Name"},
+        barmode="group",
+        title="Aspect Critique Results",
+    )
+
+    return {
+        "Aspect Scores": [
+            {"Aspect": aspect, "Score": result_df[aspect].mean()}
+            for aspect in aspects + [aspect.name for aspect in custom_aspects]
+        ]
+    }, fig

validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -0,0 +1,133 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import warnings
+
+import plotly.express as px
+from datasets import Dataset
+from ragas import evaluate
+from ragas.metrics import context_entity_recall
+
+from validmind import tags, tasks
+
+from .utils import get_renamed_columns
+
+
+@tags("ragas", "llm", "retrieval_performance")
+@tasks("text_qa", "text_generation", "text_summarization")
+def ContextEntityRecall(
+    dataset,
+    contexts_column: str = "contexts",
+    ground_truth_column: str = "ground_truth",
+):
+    """
+    Evaluates the context entity recall for dataset entries and visualizes the results.
+
+    ### Overview
+
+    This metric gives the measure of recall of the retrieved context, based on the
+    number of entities present in both `ground_truths` and `contexts` relative to the
+    number of entities present in the `ground_truths` alone. Simply put, it is a measure
+    of what fraction of entities are recalled from `ground_truths`. This metric is
+    useful in fact-based use cases like tourism help desk, historical QA, etc. This
+    metric can help evaluate the retrieval mechanism for entities, based on comparison
+    with entities present in `ground_truths`, because in cases where entities matter,
+    we need the `contexts` which cover them.
+
+    ### Formula
+
+    To compute this metric, we use two sets, $GE$ and $CE$, representing the set of
+    entities present in `ground_truths` and set of entities present in `contexts`
+    respectively. We then take the number of elements in intersection of these sets and
+    divide it by the number of elements present in the $GE$, given by the formula:
+
+    $$
+    \\text{context entity recall} = \\frac{| CE \\cap GE |}{| GE |}
+    $$
+
+    ### Configuring Columns
+
+    This metric requires the following columns in your dataset:
+    - `contexts` (List[str]): A list of text contexts which will be evaluated to make
+    sure if they contain the entities present in the ground truth.
+    - `ground_truth` (str): The ground truth text from which the entities will be
+    extracted and compared with the entities in the `contexts`.
+
+    If the above data is not in the appropriate column, you can specify different column
+    names for these fields using the parameters `contexts_column`, and `ground_truth_column`.
+
+    For example, if your dataset has this data stored in different columns, you can
+    pass the following parameters:
+    ```python
+    {
+        "contexts_column": "context_info"
+        "ground_truth_column": "my_ground_truth_col",
+    }
+    ```
+
+    If the data is stored as a dictionary in another column, specify the column and key
+    like this:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "contexts_column": f"{pred_col}.contexts",
+        "ground_truth_column": "my_ground_truth_col",
+    }
+    ```
+
+    For more complex situations, you can use a function to extract the data:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "contexts_column": lambda row: [row[pred_col]["context_message"]],
+        "ground_truth_column": "my_ground_truth_col",
+    }
+    ```
+    """
+    warnings.filterwarnings(
+        "ignore",
+        category=FutureWarning,
+        message="promote has been superseded by promote_options='default'.",
+    )
+
+    required_columns = {
+        "ground_truth": ground_truth_column,
+        "contexts": contexts_column,
+    }
+
+    df = get_renamed_columns(dataset.df, required_columns)
+
+    result_df = evaluate(
+        Dataset.from_pandas(df),
+        metrics=[context_entity_recall],
+    ).to_pandas()
+
+    fig_histogram = px.histogram(
+        x=result_df["context_entity_recall"].to_list(), nbins=10
+    )
+    fig_box = px.box(x=result_df["context_entity_recall"].to_list())
+
+    return (
+        {
+            "Scores": result_df[
+                [
+                    "contexts",
+                    "ground_truth",
+                    "context_entity_recall",
+                ]
+            ],
+            "Aggregate Scores": [
+                {
+                    "Mean Score": result_df["context_entity_recall"].mean(),
+                    "Median Score": result_df["context_entity_recall"].median(),
+                    "Max Score": result_df["context_entity_recall"].max(),
+                    "Min Score": result_df["context_entity_recall"].min(),
+                    "Standard Deviation": result_df["context_entity_recall"].std(),
+                    "Count": len(result_df),
+                }
+            ],
+        },
+        fig_histogram,
+        fig_box,
+    )