validmind 2.5.8__py3-none-any.whl → 2.5.15__py3-none-any.whl

validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py

@@ -61,54 +61,49 @@ def random_insertion(word_list):
 
 class StabilityAnalysisRandomNoise(StabilityAnalysis):
     """
-    Evaluate robustness of embeddings models to random noise introduced by using
-    a `probability` parameter to choose random locations in the text to apply
-    random perturbations. These perturbations include:
-
-    - Swapping two adjacent words
-    - Introducing a random typo in a word
-    - Deleting a random word
-    - Inserting a random word at a random position
-
-    **Purpose:**
-    The purpose of the stability analysis is to evaluate the robustness of a text embeddings model to random noise.
-    Random perturbations such as swapping adjacent words, introducing random typos, deleting random words, or inserting
-    random words at random positions in the text are introduced, to gauge the model's performance and stability.
-
-    **Test Mechanism:**
-    The test mechanism includes a series of function-defined random perturbations like swapping two words
-    `random_swap`, introducing a typo in a word `introduce_typo`, deleting a random word `random_deletion`, and
-    inserting a random word at a random position `random_insertion`. A probability parameter determines the likelihood
-    or frequency of these perturbations within the text data.
-
-    The `perturb_data` function initally tokenizes the string data based on spaces then applies selected random
-    perturbations based on the provided probability for each word in the text.
-
-    **Signs of High Risk:**
-    - High error rates in model predictions or classifications after the introduction of the random noise
-    - Greater sensitivity to certain types and degrees of noise such as typographical errors, insertion or deletion of
-    words
-    - Significant change in loss function or accuracy metric
-    - Inconsistency in model outputs for slightly perturbed inputs
-
-    **Strengths:**
-    - Measures model robustness against noise thereby reflecting real-world scenarios where data may contain errors or
+    Assesses the robustness of text embeddings models to random noise introduced via text perturbations.
+
+    ### Purpose
+
+    The purpose of this test is to evaluate the robustness of a text embeddings model to random noise. It introduces
+    perturbations such as swapping adjacent words, inserting typos, deleting words, or inserting random words within
+    the text to determine how well the model performs under such noisy conditions.
+
+    ### Test Mechanism
+
+    The test applies a series of pre-defined random perturbations to the text data. These perturbations include:
+
+    - Swapping two adjacent words using the `random_swap` function.
+    - Introducing a typo in a word using the `introduce_typo` function.
+    - Deleting a word using the `random_deletion` function.
+    - Inserting a random word at a random position using the `random_insertion` function.
+
+    A probability parameter dictates the likelihood of each perturbation being applied to the words in the text. The
+    text is initially tokenized into words, and selected perturbations are applied based on this probability.
+
+    ### Signs of High Risk
+
+    - High error rates in model predictions or classifications after the introduction of random noise.
+    - Greater sensitivity to specific types of noise, such as typographical errors or word deletions.
+    - Significant change in loss function or accuracy metrics.
+    - Inconsistent model outputs for slightly perturbed inputs.
+
+    ### Strengths
+
+    - Measures model robustness against noise, reflecting real-world scenarios where data may contain errors or
     inconsistencies.
-    - Easy to implement with adjustable perturbation severity through probability parameter.
-    - Enables identification of model sensitivity to certain noise types, providing grounding for model improvement.
+    - Easy to implement with adjustable perturbation severity through a probability parameter.
+    - Identifies model sensitivity to specific types of noise, offering insights for model improvement.
     - Useful for testing models designed to handle text data.
 
-    **Limitations:**
-    - The test might not be effective for models that have been designed with a high resistance to noise or models that
-    are inherently designed to handle such perturbations.
-    - Pseudo-randomness may not accurately represent real-world distribution of noise or typographical errors, which
-    could be biased towards certain types of errors or malware injections.
-    - Highly dependent on the probability parameter to introduce noise, with artificial adjusting required to achieve
-    an optimal balance.
-    - Only validates the model's performance against noise in input data, not its ability to capture complex language
-    structures or semantics.
-    - Does not guarantee the model's performance in new, unseen, real-world data beyond what is represented by the
-    noise-introduced test data.
+    ### Limitations
+
+    - May be ineffective for models that are inherently resistant to noise or designed to handle such perturbations.
+    - Pseudo-randomness may not accurately represent the real-world distribution of noise or typographical errors.
+    - Highly dependent on the probability parameter, requiring fine-tuning to achieve an optimal balance.
+    - Only assesses performance against noise in input data, not the ability to capture complex language structures or
+    semantics.
+    - Does not guarantee model performance on new, unseen, real-world data beyond the generated noisy test data.
     """
 
     name = "Text Embeddings Stability Analysis to Random Noise"

validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py

@@ -14,47 +14,45 @@ class StabilityAnalysisSynonyms(StabilityAnalysis):
     """
     Evaluates the stability of text embeddings models when words in test data are replaced by their synonyms randomly.
 
-    This test uses WordNet to find synonyms for words in the test dataset and
-    expects a parameter `probability` that determines the probability of swapping
-    a word with a synonym.
+    ### Purpose
 
-    **Purpose:**
     The Stability Analysis Synonyms test is designed to gauge the robustness and stability of an embeddings model on
     text-based data. The test does so by introducing random word changes through replacing words in the test dataset
     with their synonyms.
 
-    **Test Mechanism:**
+    ### Test Mechanism
+
     This test utilizes WordNet to find synonyms for a given word present in the test data, replacing the original word
     with this synonym based on a given probability. The probability is defined as a parameter and determines the
     likelihood of swapping a word with its synonym. By default, this is set at 0.02 but can be adjusted based on
     specific test requirements. This methodology enables an evaluation of how such replacements can affect the model's
     performance.
 
-    **Signs of High Risk:**
+    ### Signs of High Risk
 
     - The model's performance or predictions change significantly after swapping words with their synonyms.
     - The model shows high sensitivity to small perturbations, like modifying the data with synonyms.
-    - The embeddings model fails to identify similar meanings between the original words and their synonyms, which
-    means it lacks semantic understanding.
+    - The embeddings model fails to identify similar meanings between the original words and their synonyms, indicating
+    it lacks semantic understanding.
 
-    **Strengths:**
+    ### Strengths
 
     - The test is flexible in its application. The 'probability' parameter can be adjusted based on the degree of
     synonym swapping required.
     - Efficient in gauging a model's sensitivity or robustness with respect to small changes in input data.
-    - The test can provide insights into the semantic understanding of the model as it monitors the impact of swapping
-    words with synonyms.
+    - Provides insights into the semantic understanding of the model as it monitors the impact of swapping words with
+    synonyms.
 
-    **Limitations:**
+    ### Limitations
 
     - The ability to perturb data is reliant on the availability of synonyms, limiting its efficiency.
-    - The test assumes that the synonyms provided by WordNet are accurate and interchangeable in all contexts, which
-    may not always be the case given the intricacies of language and context-specific meanings.
-    - This test does not consider the influence of multi-word expressions or phrases, as synonyms are considered at the
-    word level only.
-    - Relies solely on the WordNet corpus for synonyms, which would limit its effectiveness for specialized or
-    domain-specific jargon not included in that corpus.
-    - It does not consider the semantic role of the words in the sentence, so the swapped synonym could potentially
+    - It assumes that the synonyms provided by WordNet are accurate and interchangeable in all contexts, which may not
+    always be the case given the intricacies of language and context-specific meanings.
+    - It does not consider the influence of multi-word expressions or phrases, as synonyms are considered at the word
+    level only.
+    - Relies solely on the WordNet corpus for synonyms, limiting its effectiveness for specialized or domain-specific
+    jargon not included in that corpus.
+    - Does not consider the semantic role of the words in the sentence, meaning the swapped synonym could potentially
     alter the overall meaning of the sentence, leading to a false perception of the model's stability.
     """
 

validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py

@@ -13,15 +13,19 @@ logger = get_logger(__name__)
 
 class StabilityAnalysisTranslation(StabilityAnalysis):
     """
-    Evaluate robustness of embeddings models to noise introduced by translating
-    the original text to another language and back.
+    Evaluates robustness of text embeddings models to noise introduced by translating the original text to another
+    language and back.
 
-    **Purpose:** The purpose of this test is to assess the robustness of text embeddings models under the influence of
-    noise. The noise in this scenario is introduced by translating the original text into another language and then
-    translating it back to the original language. Any significant changes in the model's output between the original
-    and translated-then-retranslated texts can be indicators of the model's lack of robustness to noise.
+    ### Purpose
 
-    **Test Mechanism:** The test mechanism involves several steps:
+    The purpose of this test is to assess the robustness of text embeddings models under the influence of noise. The
+    noise in this scenario is introduced by translating the original text into another language and then translating it
+    back to the original language. Any significant changes in the model's output between the original and
+    translated-then-retranslated texts can be indicators of the model's lack of robustness to noise.
+
+    ### Test Mechanism
+
+    The test mechanism involves several steps:
 
     1. Initialize the Marian tokenizer and model for both source and target languages.
     2. Translate the data from the source language to the target language.
@@ -32,29 +36,29 @@ class StabilityAnalysisTranslation(StabilityAnalysis):
     The threshold of this test output would then be determined by the tolerance level of the model to these potentially
     noisy instances.
 
-    **Signs of High Risk:**
+    ### Signs of High Risk
 
-    - If there large discrepancies between the original and double-translated text, this could indicate a high level of
-    risk, signifying that the model is not robust to noise.
-    - If a translation between languages does not closely maintain the meaning and context of the original language, it
-    may suggest inadequate robustness against this type of noise.
+    - Large discrepancies between the original and double-translated text, indicating a high level of risk and a lack
+    of robustness to noise.
+    - Translations that do not closely maintain the meaning and context of the original language, suggesting inadequate
+    robustness against this type of noise.
 
-    **Strengths:**
+    ### Strengths
 
-    - This metric is an effective way to assess the model’s sensitivity and robustness to language translation noise.
-    - The use of translation as a means to introduce noise provides a realistic scenario which the model might
-    encounter in real-world situations.
-    - This metric extends beyond simple lexical changes, testing the model’s capacity to maintain semantic meaning
-    under translational perturbations.
+    - An effective way to assess the model’s sensitivity and robustness to language translation noise.
+    - Provides a realistic scenario which the model might encounter in real-world applications by using translation to
+    introduce noise.
+    - Tests the model’s capacity to maintain semantic meaning under translational perturbations, extending beyond
+    simple lexical changes.
 
-    **Limitations:**
+    ### Limitations
 
-    - Relying solely on translation-related noise for robustness testing can overlook other types of noise not
-    reflected in language translation, such as typographical errors, grammatical mistakes, or random word substitutions.
-    - Potential inaccuracies or discrepancies in the translation process itself might influence the resultant
-    robustness score, rather than reflect an inherent failing of the model being tested.
-    - The test is predominantly language-dependent, hence it might not fully capture the robustness of the model for
-    languages with fewer resources or languages that are highly dissimilar to the source language.
+    - Relies solely on translation-related noise, potentially overlooking other types of noise such as typographical
+    errors, grammatical mistakes, or random word substitutions.
+    - Inaccuracies or discrepancies in the translation process itself might influence the resultant robustness score
+    rather than reflect an inherent failing of the model.
+    - Predominantly language-dependent, thus might not fully capture robustness for languages with fewer resources or
+    those highly dissimilar to the source language.
     """
 
     name = "Text Embeddings Stability Analysis to Translation"

validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py

@@ -23,35 +23,45 @@ def TSNEComponentsPairwisePlots(
     title="t-SNE",
 ):
     """
-    Plots individual scatter plots for pairwise combinations of t-SNE components of embeddings.
-
-    **Purpose:**
-    This function creates scatter plots for each pairwise combination of t-SNE components derived from model embeddings.
-    t-SNE (t-Distributed Stochastic Neighbor Embedding) is a machine learning algorithm for dimensionality reduction that
-    is particularly well-suited for the visualization of high-dimensional datasets.
-
-    **Test Mechanism:**
-    The function begins by extracting embeddings from the provided dataset using the specified model.
-    These embeddings are then standardized to ensure that each dimension contributes equally to the distance computation.
-    Following this, the t-SNE algorithm is applied to reduce the dimensionality of the data, with the number of components
-    specified by the user. The results are plotted using Plotly, creating scatter plots for each unique pair of components
-    if more than one component is specified.
-
-    **Signs of High Risk:**
-    - If the scatter plots show overlapping clusters or indistinct groupings, it might suggest that the
-    t-SNE parameters (such as perplexity) are not optimally set for the given data, or the data itself does not exhibit clear, separable clusters.
-    - Similar plots across different pairs of components could indicate redundancy in the components generated by t-SNE,
-    suggesting that fewer dimensions might be sufficient to represent the data's structure.
-
-    **Strengths:**
-    - Provides a visual exploration tool for high-dimensional data, simplifying the detection of patterns and clusters which are not apparent in higher dimensions.
-    - Interactive plots generated by Plotly enhance user engagement and allow for a deeper dive into specific areas of the plot, aiding in detailed data analysis.
-
-    **Limitations:**
-    - The effectiveness of t-SNE is highly dependent on the choice of parameters like perplexity and the number of components,
-    which might require tuning and experimentation for optimal results.
+    Creates scatter plots for pairwise combinations of t-SNE components to visualize embeddings and highlight potential
+    clustering structures.
+
+    ### Purpose
+
+    This function creates scatter plots for each pairwise combination of t-SNE components derived from model
+    embeddings. t-SNE (t-Distributed Stochastic Neighbor Embedding) is a machine learning algorithm for dimensionality
+    reduction that is particularly well-suited for the visualization of high-dimensional datasets.
+
+    ### Test Mechanism
+
+    The function begins by extracting embeddings from the provided dataset using the specified model. These embeddings
+    are then standardized to ensure that each dimension contributes equally to the distance computation. Following
+    this, the t-SNE algorithm is applied to reduce the dimensionality of the data, with the number of components
+    specified by the user. The results are plotted using Plotly, creating scatter plots for each unique pair of
+    components if more than one component is specified.
+
+    ### Signs of High Risk
+
+    - If the scatter plots show overlapping clusters or indistinct groupings, it might suggest that the t-SNE
+    parameters (such as perplexity) are not optimally set for the given data, or the data itself does not exhibit
+    clear, separable clusters.
+    - Similar plots across different pairs of components could indicate redundancy in the components generated by
+    t-SNE, suggesting that fewer dimensions might be sufficient to represent the data's structure.
+
+    ### Strengths
+
+    - Provides a visual exploration tool for high-dimensional data, simplifying the detection of patterns and clusters
+    which are not apparent in higher dimensions.
+    - Interactive plots generated by Plotly enhance user engagement and allow for a deeper dive into specific areas of
+    the plot, aiding in detailed data analysis.
+
+    ### Limitations
+
+    - The effectiveness of t-SNE is highly dependent on the choice of parameters like perplexity and the number of
+    components, which might require tuning and experimentation for optimal results.
     - t-SNE visualizations can be misleading if interpreted without considering the stochastic nature of the algorithm;
-    two runs with the same parameters might yield different visual outputs, necessitating multiple runs for a consistent interpretation.
+    two runs with the same parameters might yield different visual outputs, necessitating multiple runs for a
+    consistent interpretation.
     """
     # Get embeddings from the dataset using the model
     embeddings = np.stack(dataset.y_pred(model))

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -49,6 +49,7 @@ def AnswerCorrectness(
     ### Configuring Columns
 
     This metric requires specific columns to be present in the dataset:
+
     - `question` (str): The text prompt or query that was input into the model.
     - `answer` (str): The text response generated by the model.
     - `ground_truth` (str): The ground truth answer that the generated answer is compared
@@ -116,9 +117,9 @@ def AnswerCorrectness(
 
     return (
         {
-            "Scores (will not be uploaded to UI)": result_df[
-                ["question", "answer", "ground_truth", "answer_correctness"]
-            ],
+            # "Scores (will not be uploaded to UI)": result_df[
+            #     ["question", "answer", "ground_truth", "answer_correctness"]
+            # ],
             "Aggregate Scores": [
                 {
                     "Mean Score": result_df["answer_correctness"].mean(),
@@ -126,7 +127,7 @@ def AnswerCorrectness(
                     "Max Score": result_df["answer_correctness"].max(),
                     "Min Score": result_df["answer_correctness"].min(),
                     "Standard Deviation": result_df["answer_correctness"].std(),
-                    "Count": len(result_df),
+                    "Count": result_df.shape[0],
                 }
             ],
         },

validmind/tests/model_validation/ragas/AnswerRelevance.py

@@ -53,6 +53,7 @@ def AnswerRelevance(
     ### Configuring Columns
 
     This metric requires the following columns in your dataset:
+
     - `question` (str): The text query that was input into the model.
     - `contexts` (List[str]): Any contextual information retrieved by the model before
     generating an answer.
@@ -120,9 +121,9 @@ def AnswerRelevance(
 
     return (
         {
-            "Scores (will not be uploaded to UI)": result_df[
-                ["question", "contexts", "answer", "answer_relevancy"]
-            ],
+            # "Scores (will not be uploaded to UI)": result_df[
+            #     ["question", "contexts", "answer", "answer_relevancy"]
+            # ],
             "Aggregate Scores": [
                 {
                     "Mean Score": result_df["answer_relevancy"].mean(),
@@ -130,7 +131,7 @@ def AnswerRelevance(
                     "Max Score": result_df["answer_relevancy"].max(),
                     "Min Score": result_df["answer_relevancy"].min(),
                     "Standard Deviation": result_df["answer_relevancy"].std(),
-                    "Count": len(result_df),
+                    "Count": result_df.shape[0],
                 }
             ],
         },

validmind/tests/model_validation/ragas/AnswerSimilarity.py

@@ -42,6 +42,7 @@ def AnswerSimilarity(
     ### Configuring Columns
 
     This metric requires the following columns in your dataset:
+
     - `answer` (str): The text response generated by the model.
     - `ground_truth` (str): The ground truth answer that the generated answer is compared
     against.
@@ -105,9 +106,9 @@ def AnswerSimilarity(
 
     return (
         {
-            "Scores (will not be uploaded to UI)": result_df[
-                ["answer", "ground_truth", "answer_similarity"]
-            ],
+            # "Scores (will not be uploaded to UI)": result_df[
+            #     ["answer", "ground_truth", "answer_similarity"]
+            # ],
             "Aggregate Scores": [
                 {
                     "Mean Score": result_df["answer_similarity"].mean(),
@@ -115,7 +116,7 @@ def AnswerSimilarity(
                     "Max Score": result_df["answer_similarity"].max(),
                     "Min Score": result_df["answer_similarity"].min(),
                     "Standard Deviation": result_df["answer_similarity"].std(),
-                    "Count": len(result_df),
+                    "Count": result_df.shape[0],
                 }
             ],
         },

validmind/tests/model_validation/ragas/AspectCritique.py

@@ -11,6 +11,8 @@ from validmind import tags, tasks
 
 from .utils import get_ragas_config, get_renamed_columns
 
+LOWER_IS_BETTER_ASPECTS = ["harmfulness", "maliciousness"]
+
 
 @tags("ragas", "llm", "qualitative")
 @tasks("text_summarization", "text_generation", "text_qa")
@@ -149,6 +151,11 @@ def AspectCritique(
         Dataset.from_pandas(df), metrics=all_aspects, **get_ragas_config()
     ).to_pandas()
 
+    # reverse the score for aspects where lower is better
+    for aspect in LOWER_IS_BETTER_ASPECTS:
+        if aspect in result_df.columns:
+            result_df[aspect] = 1 - result_df[aspect]
+
     df_melted = result_df.melt(
         id_vars=["question", "answer", "contexts"],
         value_vars=[aspect.name for aspect in all_aspects],

validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -47,6 +47,7 @@ def ContextEntityRecall(
     ### Configuring Columns
 
     This metric requires the following columns in your dataset:
+
     - `contexts` (List[str]): A list of text contexts which will be evaluated to make
     sure if they contain the entities present in the ground truth.
     - `ground_truth` (str): The ground truth text from which the entities will be
@@ -113,13 +114,13 @@ def ContextEntityRecall(
 
     return (
         {
-            "Scores (will not be uploaded to UI)": result_df[
-                [
-                    "contexts",
-                    "ground_truth",
-                    "context_entity_recall",
-                ]
-            ],
+            # "Scores (will not be uploaded to UI)": result_df[
+            #     [
+            #         "contexts",
+            #         "ground_truth",
+            #         "context_entity_recall",
+            #     ]
+            # ],
             "Aggregate Scores": [
                 {
                     "Mean Score": result_df["context_entity_recall"].mean(),
@@ -127,7 +128,7 @@ def ContextEntityRecall(
                     "Max Score": result_df["context_entity_recall"].max(),
                     "Min Score": result_df["context_entity_recall"].min(),
                     "Standard Deviation": result_df["context_entity_recall"].std(),
-                    "Count": len(result_df),
+                    "Count": result_df.shape[0],
                 }
             ],
         },

validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -40,6 +40,7 @@ def ContextPrecision(
     ### Configuring Columns
 
     This metric requires the following columns in your dataset:
+
     - `question` (str): The text query that was input into the model.
     - `contexts` (List[str]): A list of text contexts which are retrieved and which
     will be evaluated to make sure they contain relevant info in the correct order.
@@ -107,9 +108,9 @@ def ContextPrecision(
 
     return (
         {
-            "Scores (will not be uploaded to UI)": result_df[
-                ["question", "contexts", "ground_truth", "context_precision"]
-            ],
+            # "Scores (will not be uploaded to UI)": result_df[
+            #     ["question", "contexts", "ground_truth", "context_precision"]
+            # ],
             "Aggregate Scores": [
                 {
                     "Mean Score": result_df["context_precision"].mean(),
@@ -117,7 +118,7 @@ def ContextPrecision(
                     "Max Score": result_df["context_precision"].max(),
                     "Min Score": result_df["context_precision"].min(),
                     "Standard Deviation": result_df["context_precision"].std(),
-                    "Count": len(result_df),
+                    "Count": result_df.shape[0],
                 }
             ],
         },

validmind/tests/model_validation/ragas/ContextRecall.py

@@ -40,6 +40,7 @@ def ContextRecall(
     ### Configuring Columns
 
     This metric requires the following columns in your dataset:
+
     - `question` (str): The text query that was input into the model.
     - `contexts` (List[str]): A list of text contexts which are retrieved and which
     will be evaluated to make sure they contain all items in the ground truth.
@@ -107,9 +108,9 @@ def ContextRecall(
 
     return (
         {
-            "Scores (will not be uploaded to UI)": result_df[
-                ["question", "contexts", "ground_truth", "context_recall"]
-            ],
+            # "Scores (will not be uploaded to UI)": result_df[
+            #     ["question", "contexts", "ground_truth", "context_recall"]
+            # ],
             "Aggregate Scores": [
                 {
                     "Mean Score": result_df["context_recall"].mean(),
@@ -117,7 +118,7 @@ def ContextRecall(
                     "Max Score": result_df["context_recall"].max(),
                     "Min Score": result_df["context_recall"].min(),
                     "Standard Deviation": result_df["context_recall"].std(),
-                    "Count": len(result_df),
+                    "Count": result_df.shape[0],
                 }
             ],
         },

validmind/tests/model_validation/ragas/Faithfulness.py

@@ -41,6 +41,7 @@ def Faithfulness(
     ### Configuring Columns
 
     This metric requires the following columns in your dataset:
+
     - `contexts` (List[str]): A list of text contexts which are retrieved to generate
     the answer.
     - `answer` (str): The response generated by the model which will be evaluated for
@@ -105,9 +106,9 @@ def Faithfulness(
 
     return (
         {
-            "Scores (will not be uploaded to UI)": result_df[
-                ["contexts", "answer", "faithfulness"]
-            ],
+            # "Scores (will not be uploaded to UI)": result_df[
+            #     ["contexts", "answer", "faithfulness"]
+            # ],
             "Aggregate Scores": [
                 {
                     "Mean Score": result_df["faithfulness"].mean(),
@@ -115,7 +116,7 @@ def Faithfulness(
                     "Max Score": result_df["faithfulness"].max(),
                     "Min Score": result_df["faithfulness"].min(),
                     "Standard Deviation": result_df["faithfulness"].std(),
-                    "Count": len(result_df),
+                    "Count": result_df.shape[0],
                 }
             ],
         },

validmind/tests/model_validation/ragas/utils.py

@@ -5,11 +5,17 @@
 import os
 
 from validmind.ai.utils import get_client_and_model
+from validmind.client_config import client_config
 
 EMBEDDINGS_MODEL = "text-embedding-3-small"
 
 
 def get_ragas_config():
+    if not client_config.can_generate_llm_test_descriptions():
+        raise ValueError(
+            "LLM based descriptions are not enabled in the current configuration."
+        )
+
     # import here since its an optional dependency
     try:
         from langchain_openai import ChatOpenAI, OpenAIEmbeddings