validmind 2.5.25__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py

@@ -3,11 +3,22 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import re
+from typing import Dict
 
-from .StabilityAnalysis import StabilityAnalysis
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
+from .utils import create_stability_analysis_result
 
-class StabilityAnalysisKeyword(StabilityAnalysis):
+
+@tags("llm", "text_data", "embeddings", "visualization")
+@tasks("feature_extraction")
+def StabilityAnalysisKeyword(
+    dataset: VMDataset,
+    model: VMModel,
+    keyword_dict: Dict[str, str],
+    mean_similarity_threshold: float = 0.7,
+):
     """
     Evaluates robustness of embedding models to keyword swaps in the test dataset.
 
@@ -49,13 +60,9 @@ class StabilityAnalysisKeyword(StabilityAnalysis):
     which might not always be the case.
     """
 
-    name = "Text Embeddings Stability Analysis to Keyword Swaps"
-    default_params = {
-        "keyword_dict": None,  # set to none by default... this must be overridden
-        **StabilityAnalysis.default_params,
-    }
+    keyword_dict = {k.lower(): v for k, v in keyword_dict.items()}
 
-    def perturb_data(self, data: str):
+    def perturb_data(data: str):
         if not isinstance(data, str):
             return data
 
@@ -63,22 +70,29 @@ class StabilityAnalysisKeyword(StabilityAnalysis):
         tokens = re.findall(r"[\w']+[.,!?;]?|[\w']+", data)
         modified_tokens = []
 
-        # lowercase all keys in the keword_dict
-        self.params["keyword_dict"] = {
-            k.lower(): v for k, v in self.params["keyword_dict"].items()
-        }
-
         for token in tokens:
             # Separate word and punctuation
             word_part = re.match(r"([\w']+)", token).group()
             punctuation_part = token[len(word_part) :]
 
             # Check if the token is a word and if it's in the dictionary
-            if token.lower() in self.params["keyword_dict"]:
+            if token.lower() in keyword_dict:
                 modified_tokens.append(
-                    self.params["keyword_dict"][word_part.lower()] + punctuation_part
+                    keyword_dict[word_part.lower()] + punctuation_part
                 )
             else:
                 modified_tokens.append(token)
 
         return " ".join(modified_tokens)
+
+    original_df = dataset.df[[dataset.text_column]]
+    perturbed_df = original_df.copy()
+    perturbed_df[dataset.text_column] = perturbed_df[dataset.text_column].map(
+        perturb_data
+    )
+
+    return create_stability_analysis_result(
+        dataset.y_pred(model),
+        model.predict(perturbed_df),
+        mean_similarity_threshold,
+    )

validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py

@@ -5,7 +5,10 @@
 import random
 import string
 
-from .StabilityAnalysis import StabilityAnalysis
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+from .utils import create_stability_analysis_result
 
 
 def random_swap(word_list):
@@ -59,7 +62,14 @@ def random_insertion(word_list):
     return word_list[:index] + [random_word] + word_list[index:]
 
 
-class StabilityAnalysisRandomNoise(StabilityAnalysis):
+@tags("llm", "text_data", "embeddings", "visualization")
+@tasks("feature_extraction")
+def StabilityAnalysisRandomNoise(
+    dataset: VMDataset,
+    model: VMModel,
+    probability: float = 0.02,
+    mean_similarity_threshold: float = 0.7,
+):
     """
     Assesses the robustness of text embeddings models to random noise introduced via text perturbations.
 
@@ -106,18 +116,10 @@ class StabilityAnalysisRandomNoise(StabilityAnalysis):
     - Does not guarantee model performance on new, unseen, real-world data beyond the generated noisy test data.
     """
 
-    name = "Text Embeddings Stability Analysis to Random Noise"
-    default_params = {
-        **StabilityAnalysis.default_params,
-        "probability": 0.02,
-    }
-
-    def perturb_data(self, data):
+    def perturb_data(data):
         if not isinstance(data, str):
             return data
 
-        probability = self.params["probability"]
-
         # Tokenize the string based on spaces
         words = data.split()
 
@@ -136,3 +138,15 @@ class StabilityAnalysisRandomNoise(StabilityAnalysis):
                     words = random_insertion(words)
 
         return " ".join(words)
+
+    original_df = dataset.df[[dataset.text_column]]
+    perturbed_df = original_df.copy()
+    perturbed_df[dataset.text_column] = perturbed_df[dataset.text_column].map(
+        perturb_data
+    )
+
+    return create_stability_analysis_result(
+        dataset.y_pred(model),
+        model.predict(perturbed_df),
+        mean_similarity_threshold,
+    )

validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py

@@ -7,10 +7,20 @@ import random
 import nltk
 from nltk.corpus import wordnet as wn
 
-from .StabilityAnalysis import StabilityAnalysis
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
+from .utils import create_stability_analysis_result
 
-class StabilityAnalysisSynonyms(StabilityAnalysis):
+
+@tags("llm", "text_data", "embeddings", "visualization")
+@tasks("feature_extraction")
+def StabilityAnalysisSynonyms(
+    dataset: VMDataset,
+    model: VMModel,
+    probability: float = 0.02,
+    mean_similarity_threshold: float = 0.7,
+):
     """
     Evaluates the stability of text embeddings models when words in test data are replaced by their synonyms randomly.
 
@@ -55,26 +65,19 @@ class StabilityAnalysisSynonyms(StabilityAnalysis):
     - Does not consider the semantic role of the words in the sentence, meaning the swapped synonym could potentially
     alter the overall meaning of the sentence, leading to a false perception of the model's stability.
     """
+    # download the nltk wordnet
+    nltk.download("wordnet", quiet=True)
 
-    name = "Text Embeddings Stability Analysis to Synonym Swaps"
-    default_params = {
-        "probability": 0.02,  # probability of swapping a word with a synonym
-        **StabilityAnalysis.default_params,
-    }
-
-    def perturb_data(self, data):
+    def perturb_data(data):
         if not isinstance(data, str):
             return data
 
-        # download the nltk wordnet
-        nltk.download("wordnet", quiet=True)
-
         words = nltk.word_tokenize(data)
         modified_words = []
 
         # For each word, check the probability and swap if needed
         for word in words:
-            if random.random() <= self.params["probability"]:
+            if random.random() <= probability:
                 # get synonyms for the word
                 synonyms = [
                     lemma.name() for syn in wn.synsets(word) for lemma in syn.lemmas()
@@ -91,3 +94,15 @@ class StabilityAnalysisSynonyms(StabilityAnalysis):
             modified_words.append(word)
 
         return " ".join(modified_words)
+
+    original_df = dataset.df[[dataset.text_column]]
+    perturbed_df = original_df.copy()
+    perturbed_df[dataset.text_column] = perturbed_df[dataset.text_column].map(
+        perturb_data
+    )
+
+    return create_stability_analysis_result(
+        dataset.y_pred(model),
+        model.predict(perturbed_df),
+        mean_similarity_threshold,
+    )

validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py

@@ -4,14 +4,24 @@
 
 from transformers import MarianMTModel, MarianTokenizer
 
+from validmind import tags, tasks
 from validmind.logging import get_logger
+from validmind.vm_models import VMDataset, VMModel
 
-from .StabilityAnalysis import StabilityAnalysis
+from .utils import create_stability_analysis_result
 
 logger = get_logger(__name__)
 
 
-class StabilityAnalysisTranslation(StabilityAnalysis):
+@tags("llm", "text_data", "embeddings", "visualization")
+@tasks("feature_extraction")
+def StabilityAnalysisTranslation(
+    dataset: VMDataset,
+    model: VMModel,
+    source_lang: str = "en",
+    target_lang: str = "fr",
+    mean_similarity_threshold: float = 0.7,
+):
     """
     Evaluates robustness of text embeddings models to noise introduced by translating the original text to another
     language and back.
@@ -45,10 +55,10 @@ class StabilityAnalysisTranslation(StabilityAnalysis):
 
     ### Strengths
 
-    - An effective way to assess the model’s sensitivity and robustness to language translation noise.
+    - An effective way to assess the model's sensitivity and robustness to language translation noise.
     - Provides a realistic scenario which the model might encounter in real-world applications by using translation to
     introduce noise.
-    - Tests the model’s capacity to maintain semantic meaning under translational perturbations, extending beyond
+    - Tests the model's capacity to maintain semantic meaning under translational perturbations, extending beyond
     simple lexical changes.
 
     ### Limitations
@@ -60,47 +70,66 @@ class StabilityAnalysisTranslation(StabilityAnalysis):
     - Predominantly language-dependent, thus might not fully capture robustness for languages with fewer resources or
     those highly dissimilar to the source language.
     """
+    # TODO: make the models and tokenizers configurable along with the max length
 
-    name = "Text Embeddings Stability Analysis to Translation"
-    default_params = {
-        "source_lang": "en",
-        "target_lang": "fr",
-        **StabilityAnalysis.default_params,
-    }
-
-    def perturb_data(self, data: str):
-        if len(data) > 512:
-            logger.info(
-                "Data length exceeds 512 tokens. Truncating data to 512 tokens."
-            )
-            data = data[:512]
-
-        source_lang = self.params["source_lang"]
-        target_lang = self.params["target_lang"]
-
+    try:
         # Initialize the Marian tokenizer and model for the source language
-        model_name = f"Helsinki-NLP/opus-mt-{source_lang}-{target_lang}"
-        model = MarianMTModel.from_pretrained(model_name)
-        tokenizer = MarianTokenizer.from_pretrained(model_name)
+        translate_model_name = f"Helsinki-NLP/opus-mt-{source_lang}-{target_lang}"
+        translate_model = MarianMTModel.from_pretrained(translate_model_name)
+        translate_tokenizer = MarianTokenizer.from_pretrained(translate_model_name)
 
         # Initialize the Marian tokenizer and model for the target language
-        model_name_reverse = f"Helsinki-NLP/opus-mt-{target_lang}-{source_lang}"
-        model_reverse = MarianMTModel.from_pretrained(model_name_reverse)
-        tokenizer_reverse = MarianTokenizer.from_pretrained(model_name_reverse)
-
-        # Translate to the target language
-        encoded = tokenizer.encode(data, return_tensors="pt", add_special_tokens=True)
-        decoded = tokenizer.decode(model.generate(encoded)[0], skip_special_tokens=True)
+        reverse_model_name = f"Helsinki-NLP/opus-mt-{target_lang}-{source_lang}"
+        reverse_model = MarianMTModel.from_pretrained(reverse_model_name)
+        reverse_tokenizer = MarianTokenizer.from_pretrained(reverse_model_name)
+    except Exception as e:
+        logger.error(f"Error initializing translation models: {str(e)}")
+        raise e
+
+    # Truncate input if too long (Marian models typically have max length of 512)
+    max_length = 512
+
+    def translate_data(data: str):
+        encoded = translate_tokenizer.encode(
+            data[:1024],  # Truncate input text to avoid extremely long sequences
+            return_tensors="pt",
+            max_length=max_length,
+            truncation=True,
+            padding=True,
+        )
+        translated = translate_model.generate(
+            encoded, max_length=max_length, num_beams=2, early_stopping=True
+        )
+        decoded = translate_tokenizer.decode(translated[0], skip_special_tokens=True)
 
-        # Translate back to the source language
-        reverse_encoded = tokenizer_reverse.encode(
+        reverse_encoded = reverse_tokenizer.encode(
             decoded,
             return_tensors="pt",
-            add_special_tokens=True,
+            max_length=max_length,
+            truncation=True,
+            padding=True,
         )
-        reverse_decoded = tokenizer_reverse.decode(
-            model_reverse.generate(reverse_encoded)[0],
-            skip_special_tokens=True,
+        reverse_translated = reverse_model.generate(
+            reverse_encoded, max_length=max_length, num_beams=2, early_stopping=True
         )
 
-        return reverse_decoded
+        return reverse_tokenizer.decode(reverse_translated[0], skip_special_tokens=True)
+
+    def perturb_data(data):
+        try:
+            return translate_data(data)
+        except Exception as e:
+            logger.error(f"Error translating data: {str(e)}")
+            return data
+
+    original_df = dataset.df[[dataset.text_column]]
+    perturbed_df = original_df.copy()
+    perturbed_df[dataset.text_column] = perturbed_df[dataset.text_column].map(
+        perturb_data
+    )
+
+    return create_stability_analysis_result(
+        dataset.y_pred(model),
+        model.predict(perturbed_df),
+        mean_similarity_threshold,
+    )

validmind/tests/model_validation/embeddings/utils.py

@@ -0,0 +1,53 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+import plotly.express as px
+from sklearn.metrics.pairwise import cosine_similarity
+
+
+def create_stability_analysis_result(
+    original_embeddings,
+    perturbed_embeddings,
+    mean_similarity_threshold=0.7,
+):
+    # Compute cosine similarities between original and perturbed embeddings
+    similarities = cosine_similarity(
+        original_embeddings, perturbed_embeddings
+    ).diagonal()
+
+    mean = np.mean(similarities)
+    passed = mean > mean_similarity_threshold
+
+    return (
+        [
+            {
+                "Mean Similarity": mean,
+                "Min Similarity": np.min(similarities),
+                "Max Similarity": np.max(similarities),
+                "Median Similarity": np.median(similarities),
+                "Std Similarity": np.std(similarities),
+                "Pass/Fail": "Pass" if passed else "Fail",
+            }
+        ],
+        px.histogram(
+            x=similarities.flatten(),
+            nbins=100,
+            title="Cosine Similarity Distribution",
+            labels={"x": "Cosine Similarity"},
+        ),
+        px.density_contour(
+            x=similarities.flatten(),
+            nbinsx=100,
+            title="Cosine Similarity Density",
+            labels={"x": "Cosine Similarity"},
+            marginal_x="histogram",
+        ),
+        px.box(
+            x=similarities.flatten(),
+            labels={"x": "Cosine Similarity"},
+            title="Cosine Similarity Box Plot",
+        ),
+        passed,
+    )

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -14,23 +14,26 @@ from .utils import get_ragas_config, get_renamed_columns
 
 try:
     from ragas import evaluate
-    from ragas.metrics import answer_correctness
+    from ragas.metrics import AnswerCorrectness as answer_correctness
 except ImportError as e:
-    raise MissingDependencyError(
-        "Missing required package `ragas` for AnswerCorrectness. "
-        "Please run `pip install validmind[llm]` to use LLM tests",
-        required_dependencies=["ragas"],
-        extra="llm",
-    ) from e
+    if "ragas" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `ragas` for AnswerCorrectness. "
+            "Please run `pip install validmind[llm]` to use LLM tests",
+            required_dependencies=["ragas"],
+            extra="llm",
+        ) from e
+
+    raise e
 
 
 @tags("ragas", "llm")
 @tasks("text_qa", "text_generation", "text_summarization")
 def AnswerCorrectness(
     dataset,
-    question_column="question",
-    answer_column="answer",
-    ground_truth_column="ground_truth",
+    user_input_column="user_input",
+    response_column="response",
+    reference_column="reference",
 ):
     """
     Evaluates the correctness of answers in a dataset with respect to the provided ground
@@ -62,9 +65,9 @@ def AnswerCorrectness(
 
     This metric requires specific columns to be present in the dataset:
 
-    - `question` (str): The text prompt or query that was input into the model.
-    - `answer` (str): The text response generated by the model.
-    - `ground_truth` (str): The ground truth answer that the generated answer is compared
+    - `user_input` (str): The text prompt or query that was input into the model.
+    - `response` (str): The text response generated by the model.
+    - `reference` (str): The ground truth answer that the generated answer is compared
     against.
 
     If the above data is not in the appropriate column, you can specify different column
@@ -75,9 +78,9 @@ def AnswerCorrectness(
     pass the following parameters:
     ```python
     params = {
-        "question_column": "input_text",
-        "answer_column": "output_text",
-        "ground_truth_column": "human_answer",
+        "user_input_column": "input_text",
+        "response_column": "output_text",
+        "reference_column": "human_answer",
     }
     ```
 
@@ -86,8 +89,8 @@ def AnswerCorrectness(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "answer_column": f"{pred_col}.generated_answer",
-        "ground_truth_column": f"{pred_col}.contexts",
+        "response_column": f"{pred_col}.generated_answer",
+        "reference_column": f"{pred_col}.contexts",
     }
     ```
 
@@ -95,8 +98,8 @@ def AnswerCorrectness(
     ```python
     pred_col = dataset.prediction_column(model)
     params = {
-        "answer_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
-        "ground_truth_column": lambda row: [row[pred_col]["context_message"]],
+        "response_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
+        "reference_column": lambda row: [row[pred_col]["context_message"]],
     }
     ```
     """
@@ -107,32 +110,34 @@ def AnswerCorrectness(
     )
 
     required_columns = {
-        "question": question_column,
-        "answer": answer_column,
-        "ground_truth": ground_truth_column,
+        "user_input": user_input_column,
+        "response": response_column,
+        "reference": reference_column,
     }
 
     df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[answer_correctness], **get_ragas_config()
+        Dataset.from_pandas(df), metrics=[answer_correctness()], **get_ragas_config()
     ).to_pandas()
 
-    fig_histogram = px.histogram(x=result_df["answer_correctness"].to_list(), nbins=10)
-    fig_box = px.box(x=result_df["answer_correctness"].to_list())
+    score_column = "answer_correctness"
+
+    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
+    fig_box = px.box(x=result_df[score_column].to_list())
 
     return (
         {
-            # "Scores (will not be uploaded to UI)": result_df[
+            # "Scores (will not be uploaded to ValidMind Platform)": result_df[
             #     ["question", "answer", "ground_truth", "answer_correctness"]
             # ],
             "Aggregate Scores": [
                 {
-                    "Mean Score": result_df["answer_correctness"].mean(),
-                    "Median Score": result_df["answer_correctness"].median(),
-                    "Max Score": result_df["answer_correctness"].max(),
-                    "Min Score": result_df["answer_correctness"].min(),
-                    "Standard Deviation": result_df["answer_correctness"].std(),
+                    "Mean Score": result_df[score_column].mean(),
+                    "Median Score": result_df[score_column].median(),
+                    "Max Score": result_df[score_column].max(),
+                    "Min Score": result_df[score_column].min(),
+                    "Standard Deviation": result_df[score_column].std(),
                     "Count": result_df.shape[0],
                 }
             ],

validmind/tests/model_validation/ragas/{AspectCritique.py → AspectCritic.py}

@@ -14,7 +14,7 @@ from .utils import get_ragas_config, get_renamed_columns
 
 try:
     from ragas import evaluate
-    from ragas.metrics import AspectCritic
+    from ragas.metrics import AspectCritic as aspect_critic
     from ragas.metrics._aspect_critic import (
         coherence,
         conciseness,
@@ -23,24 +23,27 @@ try:
         maliciousness,
     )
 except ImportError as e:
-    raise MissingDependencyError(
-        "Missing required package `ragas` for AspectCritique. "
-        "Please run `pip install validmind[llm]` to use LLM tests",
-        required_dependencies=["ragas"],
-        extra="llm",
-    ) from e
+    if "ragas" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `ragas` for AspectCritic. "
+            "Please run `pip install validmind[llm]` to use LLM tests",
+            required_dependencies=["ragas"],
+            extra="llm",
+        ) from e
+
+    raise e
 
 LOWER_IS_BETTER_ASPECTS = ["harmfulness", "maliciousness"]
 
 
 @tags("ragas", "llm", "qualitative")
 @tasks("text_summarization", "text_generation", "text_qa")
-def AspectCritique(
+def AspectCritic(
     dataset,
-    question_column="question",
-    answer_column="answer",
-    contexts_column="contexts",
-    aspects: list = [  # noqa: B006 this is fine as immutable default since it never gets modified
+    user_input_column="user_input",
+    response_column="response",
+    retrieved_contexts_column=None,
+    aspects: list = [
         "coherence",
         "conciseness",
         "correctness",
@@ -62,13 +65,13 @@ def AspectCritique(
 
     ### Inputs and Outputs:
 
-    The input to this metric is a dataset containing the input `question` (prompt to the LLM)
-    and the `answer` (text generated by the LLM). Any retrieved `contexts` can also be
+    The input to this metric is a dataset containing the input `user_input` (prompt to the LLM)
+    and the `response` (text generated by the LLM). Any retrieved `retrieved_contexts` can also be
     included to enhance the evaluation.
 
-    The `question_column`, `answer_column`, and `contexts_column` parameters can be used to
+    The `user_input_column`, `response_column`, and `retrieved_contexts_column` parameters can be used to
     specify the names or sources for the data that this metric will evaluate if the dataset
-    does not contain the required columns `question`, `answer`, and `contexts`.
+    does not contain the required columns `user_input`, `response`, and `retrieved_contexts`.
 
     By default, the aspects evaluated are harmfulness, maliciousness, coherence,
     correctness, and conciseness. To change the aspects evaluated, the `aspects` parameter
@@ -87,17 +90,17 @@ def AspectCritique(
     ### Examples:
 
     - **Mapping to Required Columns:** If the dataset does not contain the columns required
-    to run this metric (i.e., `question`, `answer`, and `contexts`), the
+    to run this metric (i.e., `user_input`, `response`, and `retrieved_contexts`), the
 
     ```python
     pred_col = my_vm_dataset.prediction_column(my_vm_model)
     run_test(
-        "validmind.model_validation.ragas.AspectCritique",
+        "validmind.model_validation.ragas.AspectCritic",
         inputs={"dataset": my_vm_dataset},
         params={
-            "question_column": "input_prompt",
-            "answer_column": f"{pred_col}.llm_output",
-            "contexts_column": lambda row: [row[pred_col]["context_message"]],
+            "user_input_column": "input_prompt",
+            "response_column": f"{pred_col}.llm_output",
+            "retrieved_contexts_column": "retrieval_model_prediction",
         },
     )
     ```
@@ -110,7 +113,7 @@ def AspectCritique(
 
     ```python
     run_test(
-        "validmind.model_validation.ragas.AspectCritique",
+        "validmind.model_validation.ragas.AspectCritic",
         inputs={"dataset": my_vm_dataset},
         params={
             "additional_aspects": [
@@ -135,16 +138,18 @@ def AspectCritique(
     )
 
     required_columns = {
-        "question": question_column,
-        "answer": answer_column,
-        "contexts": contexts_column,
+        "user_input": user_input_column,
+        "response": response_column,
     }
 
+    if retrieved_contexts_column:
+        required_columns["retrieved_contexts"] = retrieved_contexts_column
+
     df = get_renamed_columns(dataset._df, required_columns)
 
     custom_aspects = (
         [
-            AspectCritic(name=name, definition=description)
+            aspect_critic(name=name, definition=description)
             for name, description in additional_aspects
         ]
         if additional_aspects
@@ -162,7 +167,8 @@ def AspectCritique(
             result_df[aspect] = 1 - result_df[aspect]
 
     df_melted = result_df.melt(
-        id_vars=["question", "answer", "contexts"],
+        id_vars=["user_input", "response"]
+        + (["retrieved_contexts"] if retrieved_contexts_column else []),
         value_vars=[aspect.name for aspect in all_aspects],
         var_name="Metric",
         value_name="Result",