validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/tests/model_validation/ragas/Faithfulness.py

@@ -14,22 +14,26 @@ from .utils import get_ragas_config, get_renamed_columns
 
 try:
     from ragas import evaluate
-    from ragas.metrics import faithfulness
+    from ragas.metrics import Faithfulness as faithfulness
 except ImportError as e:
-    raise MissingDependencyError(
-        "Missing required package `ragas` for Faithfulness. "
-        "Please run `pip install validmind[llm]` to use LLM tests",
-        required_dependencies=["ragas"],
-        extra="llm",
-    ) from e
+    if "ragas" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `ragas` for Faithfulness. "
+            "Please run `pip install validmind[llm]` to use LLM tests",
+            required_dependencies=["ragas"],
+            extra="llm",
+        ) from e
+
+    raise e
 
 
 @tags("ragas", "llm", "rag_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
 def Faithfulness(
     dataset,
-    answer_column="answer",
-    contexts_column="contexts",
+    user_input_column="user_input",
+    response_column="response",
+    retrieved_contexts_column="retrieved_contexts",
 ):  # noqa
     """
     Evaluates the faithfulness of the generated answers with respect to retrieved contexts.
@@ -54,20 +58,23 @@ def Faithfulness(
 
     This metric requires the following columns in your dataset:
 
-    - `contexts` (List[str]): A list of text contexts which are retrieved to generate
+    - `user_input` (str): The user input that the model is responding to.
+    - `retrieved_contexts` (List[str]): A list of text contexts which are retrieved to generate
     the answer.
-    - `answer` (str): The response generated by the model which will be evaluated for
+    - `response` (str): The response generated by the model which will be evaluated for
     faithfulness against the given contexts.
 
     If the above data is not in the appropriate column, you can specify different column
-    names for these fields using the parameters `contexts_column` and `answer_column`.
+    names for these fields using the parameters `retrieved_contexts_column` and
+    `response_column`.
 
     For example, if your dataset has this data stored in different columns, you can
     pass the following parameters:
     ```python
     {
-        "contexts_column": "context_info"
-        "answer_column": "my_answer_col",
+        "retrieved_contexts_column": "context_info",
+        "response_column": "my_answer_col",
+        "user_input_column": "user_input",
     }
     ```
 
@@ -76,8 +83,9 @@ def Faithfulness(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "contexts_column": f"{pred_col}.contexts",
-        "answer_column": f"{pred_col}.answer",
+        "retrieved_contexts_column": f"{pred_col}.retrieved_contexts",
+        "response_column": f"{pred_col}.response",
+        "user_input_column": "user_input",
     }
     ```
 
@@ -85,8 +93,9 @@ def Faithfulness(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "contexts_column": lambda row: [row[pred_col]["context_message"]],
-        "answer_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
+        "retrieved_contexts_column": lambda row: [row[pred_col]["context_message"]],
+        "response_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
+        "user_input_column": "user_input",
     }
     ```
     """
@@ -97,31 +106,34 @@ def Faithfulness(
     )
 
     required_columns = {
-        "answer": answer_column,
-        "contexts": contexts_column,
+        "response": response_column,
+        "retrieved_contexts": retrieved_contexts_column,
+        "user_input": user_input_column,
     }
 
     df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[faithfulness], **get_ragas_config()
+        Dataset.from_pandas(df), metrics=[faithfulness()], **get_ragas_config()
     ).to_pandas()
 
-    fig_histogram = px.histogram(x=result_df["faithfulness"].to_list(), nbins=10)
-    fig_box = px.box(x=result_df["faithfulness"].to_list())
+    score_column = "faithfulness"
+
+    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
+    fig_box = px.box(x=result_df[score_column].to_list())
 
     return (
         {
-            # "Scores (will not be uploaded to UI)": result_df[
-            #     ["contexts", "answer", "faithfulness"]
+            # "Scores (will not be uploaded to ValidMind Platform)": result_df[
+            #     ["retrieved_contexts", "response", "faithfulness"]
             # ],
             "Aggregate Scores": [
                 {
-                    "Mean Score": result_df["faithfulness"].mean(),
-                    "Median Score": result_df["faithfulness"].median(),
-                    "Max Score": result_df["faithfulness"].max(),
-                    "Min Score": result_df["faithfulness"].min(),
-                    "Standard Deviation": result_df["faithfulness"].std(),
+                    "Mean Score": result_df[score_column].mean(),
+                    "Median Score": result_df[score_column].median(),
+                    "Max Score": result_df[score_column].max(),
+                    "Min Score": result_df[score_column].min(),
+                    "Standard Deviation": result_df[score_column].std(),
                     "Count": result_df.shape[0],
                 }
             ],

validmind/tests/model_validation/ragas/NoiseSensitivity.py

@@ -14,23 +14,30 @@ from .utils import get_ragas_config, get_renamed_columns
 
 try:
     from ragas import evaluate
-    from ragas.metrics import noise_sensitivity_relevant
+    from ragas.metrics import NoiseSensitivity as noise_sensitivity
 except ImportError as e:
-    raise MissingDependencyError(
-        "Missing required package `ragas` for NoiseSensitivity. "
-        "Please run `pip install validmind[llm]` to use LLM tests",
-        required_dependencies=["ragas"],
-        extra="llm",
-    ) from e
+    if "ragas" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `ragas` for NoiseSensitivity. "
+            "Please run `pip install validmind[llm]` to use LLM tests",
+            required_dependencies=["ragas"],
+            extra="llm",
+        ) from e
+
+    raise e
+
+VALID_FOCUS_VALUES = ["relevant", "irrelevant"]
 
 
 @tags("ragas", "llm", "rag_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
 def NoiseSensitivity(
     dataset,
-    answer_column="answer",
-    contexts_column="contexts",
-    ground_truth_column="ground_truth",
+    response_column="response",
+    retrieved_contexts_column="retrieved_contexts",
+    reference_column="reference",
+    focus="relevant",
+    user_input_column="user_input",
 ):
     """
     Assesses the sensitivity of a Large Language Model (LLM) to noise in retrieved context by measuring how often it
@@ -58,20 +65,22 @@ def NoiseSensitivity(
 
     This metric requires the following columns in your dataset:
 
-    - `contexts` (List[str]): A list of text contexts which are retrieved to generate
+    - `retrieved_contexts` (List[str]): A list of text contexts which are retrieved to generate
     the answer.
-    - `answer` (str): The response generated by the model
-    - `ground_truth` (str): The "correct" answer to the question
-
+    - `response` (str): The response generated by the model
+    - `reference` (str): The "correct" answer to the question
+    - `user_input` (str): The user input question
     If the above data is not in the appropriate column, you can specify different column
-    names for these fields using the parameters `contexts_column` and `answer_column`.
+    names for these fields using the parameters `retrieved_contexts_column` and `response_column`.
 
     For example, if your dataset has this data stored in different columns, you can
     pass the following parameters:
     ```python
     {
-        "contexts_column": "context_info"
-        "answer_column": "my_answer_col",
+        "retrieved_contexts_column": "context_info",
+        "response_column": "my_answer_col",
+        "reference_column": "reference",
+        "user_input_column": "user_input",
     }
     ```
 
@@ -80,8 +89,10 @@ def NoiseSensitivity(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "contexts_column": f"{pred_col}.contexts",
-        "answer_column": f"{pred_col}.answer",
+        "reference_column": "reference",
+        "retrieved_contexts_column": f"{pred_col}.retrieved_contexts",
+        "response_column": f"{pred_col}.response",
+        "user_input_column": "user_input",
     }
     ```
 
@@ -89,8 +100,10 @@ def NoiseSensitivity(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "contexts_column": lambda row: [row[pred_col]["context_message"]],
-        "answer_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
+        "reference_column": "reference",
+        "retrieved_contexts_column": lambda row: [row[pred_col]["context_message"]],
+        "response_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
+        "user_input_column": "user_input",
     }
 
     ### Signs of High Risk
@@ -118,37 +131,48 @@ def NoiseSensitivity(
         message="promote has been superseded by promote_options='default'.",
     )
 
+    if focus not in VALID_FOCUS_VALUES:
+        raise ValueError(
+            f"Invalid focus parameter: '{focus}'. "
+            f"Must be one of: {VALID_FOCUS_VALUES}"
+        )
+
     required_columns = {
-        "answer": answer_column,
-        "contexts": contexts_column,
-        "ground_truth": ground_truth_column,
+        "response": response_column,
+        "retrieved_contexts": retrieved_contexts_column,
+        "reference": reference_column,
+        "user_input": user_input_column,
     }
 
     df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
         Dataset.from_pandas(df),
-        metrics=[noise_sensitivity_relevant],
+        metrics=[noise_sensitivity(focus=focus)],
         **get_ragas_config(),
     ).to_pandas()
 
+    score_column = f"noise_sensitivity_{focus}"
+
     fig_histogram = px.histogram(
-        x=result_df["noise_sensitivity_relevant"].to_list(), nbins=10
+        x=result_df[score_column].to_list(),
+        nbins=10,
+        title=f"Noise Sensitivity ({focus})",
+    )
+    fig_box = px.box(
+        x=result_df[score_column].to_list(),
+        title=f"Noise Sensitivity Distribution ({focus})",
     )
-    fig_box = px.box(x=result_df["noise_sensitivity_relevant"].to_list())
 
     return (
         {
-            # "Scores (will not be uploaded to UI)": result_df[
-            #     ["contexts", "answer", "ground_truth", "noise_sensitivity_relevant"]
-            # ],
             "Aggregate Scores": [
                 {
-                    "Mean Score": result_df["noise_sensitivity_relevant"].mean(),
-                    "Median Score": result_df["noise_sensitivity_relevant"].median(),
-                    "Max Score": result_df["noise_sensitivity_relevant"].max(),
-                    "Min Score": result_df["noise_sensitivity_relevant"].min(),
-                    "Standard Deviation": result_df["noise_sensitivity_relevant"].std(),
+                    "Mean Score": result_df[score_column].mean(),
+                    "Median Score": result_df[score_column].median(),
+                    "Max Score": result_df[score_column].max(),
+                    "Min Score": result_df[score_column].min(),
+                    "Standard Deviation": result_df[score_column].std(),
                     "Count": result_df.shape[0],
                 }
             ],

validmind/tests/model_validation/ragas/{AnswerRelevance.py → ResponseRelevancy.py}

@@ -14,36 +14,39 @@ from .utils import get_ragas_config, get_renamed_columns
 
 try:
     from ragas import evaluate
-    from ragas.metrics import answer_relevancy
+    from ragas.metrics import ResponseRelevancy as response_relevancy
 except ImportError as e:
-    raise MissingDependencyError(
-        "Missing required package `ragas` for AnswerRelevance. "
-        "Please run `pip install validmind[llm]` to use LLM tests",
-        required_dependencies=["ragas"],
-        extra="llm",
-    ) from e
+    if "ragas" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `ragas` for AnswerRelevance. "
+            "Please run `pip install validmind[llm]` to use LLM tests",
+            required_dependencies=["ragas"],
+            extra="llm",
+        ) from e
+
+    raise e
 
 
 @tags("ragas", "llm", "rag_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
-def AnswerRelevance(
+def ResponseRelevancy(
     dataset,
-    question_column="question",
-    contexts_column="contexts",
-    answer_column="answer",
+    user_input_column="user_input",
+    retrieved_contexts_column=None,
+    response_column="response",
 ):
     """
     Assesses how pertinent the generated answer is to the given prompt.
 
-    The evaluation metric, Answer Relevancy, focuses on assessing how pertinent the
+    The evaluation metric, Response Relevancy, focuses on assessing how pertinent the
     generated answer is to the given prompt. A lower score is assigned to answers that
     are incomplete or contain redundant information and higher scores indicate better
-    relevancy. This metric is computed using the `question`, the `contexts` and the
-    `answer`.
+    relevancy. This metric is computed using the `user_input`, the `retrieved_contexts`
+    and the `response`.
 
-    The Answer Relevancy is defined as the mean cosine similartiy of the original
-    `question` to a number of artifical questions, which are generated (reverse-engineered)
-    based on the `answer`:
+    The Response Relevancy is defined as the mean cosine similartiy of the original
+    `user_input` to a number of artifical questions, which are generated (reverse-engineered)
+    based on the `response`:
 
     $$
     \\text{answer relevancy} = \\frac{1}{N} \\sum_{i=1}^{N} cos(E_{g_i}, E_o)
@@ -66,10 +69,10 @@ def AnswerRelevance(
 
     This metric requires the following columns in your dataset:
 
-    - `question` (str): The text query that was input into the model.
-    - `contexts` (List[str]): Any contextual information retrieved by the model before
-    generating an answer.
-    - `answer` (str): The response generated by the model.
+    - `user_input` (str): The text query that was input into the model.
+    - `retrieved_contexts` (List[str]): Any contextual information retrieved by the model
+    before generating an answer.
+    - `response` (str): The response generated by the model.
 
     If the above data is not in the appropriate column, you can specify different column
     names for these fields using the parameters `question_column`, `answer_column`, and
@@ -79,9 +82,9 @@ def AnswerRelevance(
     pass the following parameters:
     ```python
     params = {
-        "question_column": "input_text",
-        "answer_column": "output_text",
-        "contexts_column": "context_info"
+        "user_input_column": "input_text",
+        "response_column": "output_text",
+        "retrieved_contexts_column": "context_info"
     }
     ```
 
@@ -90,8 +93,8 @@ def AnswerRelevance(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "answer_column": f"{pred_col}.generated_answer",
-        "contexts_column": f"{pred_col}.contexts",
+        "response_column": f"{pred_col}.generated_answer",
+        "retrieved_contexts_column": f"{pred_col}.contexts",
     }
     ```
 
@@ -99,8 +102,8 @@ def AnswerRelevance(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "answer_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
-        "contexts_column": lambda row: [row[pred_col]["context_message"]],
+        "response_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
+        "retrieved_contexts_column": lambda row: [row[pred_col]["context_message"]],
     }
     ```
     """
@@ -111,32 +114,40 @@ def AnswerRelevance(
     )
 
     required_columns = {
-        "question": question_column,
-        "answer": answer_column,
-        "contexts": contexts_column,
+        "user_input": user_input_column,
+        "response": response_column,
     }
 
+    if retrieved_contexts_column:
+        required_columns["retrieved_contexts"] = retrieved_contexts_column
+
     df = get_renamed_columns(dataset._df, required_columns)
 
+    metrics = [response_relevancy()]
+
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[answer_relevancy], **get_ragas_config()
+        Dataset.from_pandas(df),
+        metrics=metrics,
+        **get_ragas_config(),
     ).to_pandas()
 
-    fig_histogram = px.histogram(x=result_df["answer_relevancy"].to_list(), nbins=10)
-    fig_box = px.box(x=result_df["answer_relevancy"].to_list())
+    score_column = "answer_relevancy"
+
+    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
+    fig_box = px.box(x=result_df[score_column].to_list())
 
     return (
         {
-            # "Scores (will not be uploaded to UI)": result_df[
-            #     ["question", "contexts", "answer", "answer_relevancy"]
+            # "Scores (will not be uploaded to ValidMind Platform)": result_df[
+            #     ["user_input", "retrieved_contexts", "response", "answer_relevancy"]
             # ],
             "Aggregate Scores": [
                 {
-                    "Mean Score": result_df["answer_relevancy"].mean(),
-                    "Median Score": result_df["answer_relevancy"].median(),
-                    "Max Score": result_df["answer_relevancy"].max(),
-                    "Min Score": result_df["answer_relevancy"].min(),
-                    "Standard Deviation": result_df["answer_relevancy"].std(),
+                    "Mean Score": result_df[score_column].mean(),
+                    "Median Score": result_df[score_column].median(),
+                    "Max Score": result_df[score_column].max(),
+                    "Min Score": result_df[score_column].min(),
+                    "Standard Deviation": result_df[score_column].std(),
                     "Count": result_df.shape[0],
                 }
             ],

validmind/tests/model_validation/ragas/{AnswerSimilarity.py → SemanticSimilarity.py}

@@ -14,30 +14,33 @@ from .utils import get_ragas_config, get_renamed_columns
 
 try:
     from ragas import evaluate
-    from ragas.metrics import answer_similarity
+    from ragas.metrics import SemanticSimilarity as semantic_similarity
 except ImportError as e:
-    raise MissingDependencyError(
-        "Missing required package `ragas` for AnswerSimilarity. "
-        "Please run `pip install validmind[llm]` to use LLM tests",
-        required_dependencies=["ragas"],
-        extra="llm",
-    ) from e
+    if "ragas" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `ragas` for AnswerSimilarity. "
+            "Please run `pip install validmind[llm]` to use LLM tests",
+            required_dependencies=["ragas"],
+            extra="llm",
+        ) from e
+
+    raise e
 
 
 @tags("ragas", "llm")
 @tasks("text_qa", "text_generation", "text_summarization")
-def AnswerSimilarity(
+def SemanticSimilarity(
     dataset,
-    answer_column="answer",
-    ground_truth_column="ground_truth",
+    response_column="response",
+    reference_column="reference",
 ):
     """
-    Calculates the semantic similarity between generated answers and ground truths
+    Calculates the semantic similarity between generated responses and ground truths
 
     The concept of Answer Semantic Similarity pertains to the assessment of the semantic
     resemblance between the generated answer and the ground truth. This evaluation is
-    based on the `ground_truth` and the `answer`, with values falling within the range
-    of 0 to 1. A higher score signifies a better alignment between the generated answer
+    based on the `reference` and the `response`, with values falling within the range
+    of 0 to 1. A higher score signifies a better alignment between the generated response
     and the ground truth.
 
     Measuring the semantic similarity between answers can offer valuable insights into
@@ -55,19 +58,19 @@ def AnswerSimilarity(
 
     This metric requires the following columns in your dataset:
 
-    - `answer` (str): The text response generated by the model.
-    - `ground_truth` (str): The ground truth answer that the generated answer is compared
+    - `response` (str): The text response generated by the model.
+    - `reference` (str): The ground truth answer that the generated answer is compared
     against.
 
     If the above data is not in the appropriate column, you can specify different column
-    names for these fields using the parameters `answer_column`, and `ground_truth_column`.
+    names for these fields using the parameters `response_column`, and `reference_column`.
 
     For example, if your dataset has this data stored in different columns, you can
     pass the following parameters:
     ```python
     {
-        "answer_column": "llm_output_col",
-        "ground_truth_column": "my_ground_truth_col",
+        "response_column": "llm_output_col",
+        "reference_column": "my_ground_truth_col",
     }
     ```
 
@@ -76,8 +79,8 @@ def AnswerSimilarity(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "answer_column": f"{pred_col}.generated_answer",
-        "ground_truth_column": "my_ground_truth_col",
+        "response_column": f"{pred_col}.generated_answer",
+        "reference_column": "my_ground_truth_col",
     }
     ```
 
@@ -85,8 +88,8 @@ def AnswerSimilarity(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "answer_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
-        "ground_truth_column": "my_ground_truth_col",
+        "response_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
+        "reference_column": "my_ground_truth_col",
     }
     ```
     """
@@ -97,31 +100,33 @@ def AnswerSimilarity(
     )
 
     required_columns = {
-        "answer": answer_column,
-        "ground_truth": ground_truth_column,
+        "response": response_column,
+        "reference": reference_column,
     }
 
     df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[answer_similarity], **get_ragas_config()
+        Dataset.from_pandas(df), metrics=[semantic_similarity()], **get_ragas_config()
     ).to_pandas()
 
-    fig_histogram = px.histogram(x=result_df["answer_similarity"].to_list(), nbins=10)
-    fig_box = px.box(x=result_df["answer_similarity"].to_list())
+    score_column = "semantic_similarity"
+
+    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
+    fig_box = px.box(x=result_df[score_column].to_list())
 
     return (
         {
-            # "Scores (will not be uploaded to UI)": result_df[
-            #     ["answer", "ground_truth", "answer_similarity"]
+            # "Scores (will not be uploaded to ValidMind Platform)": result_df[
+            #     ["response", "reference", "semantic_similarity"]
             # ],
             "Aggregate Scores": [
                 {
-                    "Mean Score": result_df["answer_similarity"].mean(),
-                    "Median Score": result_df["answer_similarity"].median(),
-                    "Max Score": result_df["answer_similarity"].max(),
-                    "Min Score": result_df["answer_similarity"].min(),
-                    "Standard Deviation": result_df["answer_similarity"].std(),
+                    "Mean Score": result_df[score_column].mean(),
+                    "Median Score": result_df[score_column].median(),
+                    "Max Score": result_df[score_column].max(),
+                    "Min Score": result_df[score_column].min(),
+                    "Standard Deviation": result_df[score_column].std(),
                     "Count": result_df.shape[0],
                 }
             ],

validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py

@@ -2,15 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
+from sklearn.metrics import adjusted_mutual_info_score
 
-from sklearn import metrics
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
-from .ClusterPerformance import ClusterPerformance
 
-
-@dataclass
-class AdjustedMutualInformation(ClusterPerformance):
+@tags("sklearn", "model_performance", "clustering")
+@tasks("clustering")
+def AdjustedMutualInformation(model: VMModel, dataset: VMDataset):
     """
     Evaluates clustering model performance by measuring mutual information between true and predicted labels, adjusting
     for chance.
@@ -52,14 +52,11 @@ class AdjustedMutualInformation(ClusterPerformance):
     - The interpretability of the score can be complex as it depends on the understanding of information theory
     concepts.
     """
-
-    name = "adjusted_mutual_information"
-    required_inputs = ["model", "dataset"]
-    tasks = ["clustering"]
-    tags = [
-        "sklearn",
-        "model_performance",
+    return [
+        {
+            "Adjusted Mutual Information": adjusted_mutual_info_score(
+                labels_true=dataset.y,
+                labels_pred=dataset.y_pred(model),
+            )
+        }
     ]
-
-    def metric_info(self):
-        return {"Adjusted Mutual Information": metrics.adjusted_mutual_info_score}