validmind 2.5.15__py3-none-any.whl → 2.5.19__py3-none-any.whl

validmind/tests/model_validation/ContextualRecall.py

@@ -58,6 +58,9 @@ def ContextualRecall(dataset, model):
     - Models that effectively use infrequent words might be undervalued, as these words might not overlap as often.
     """
 
+    # download nltk data
+    nltk.download("punkt_tab", quiet=True)
+
     y_true = dataset.y
     y_pred = dataset.y_pred(model)
 

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import answer_correctness
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for AnswerCorrectness. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -88,11 +100,6 @@ def AnswerCorrectness(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import answer_correctness
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
 
     warnings.filterwarnings(
         "ignore",

validmind/tests/model_validation/ragas/AnswerRelevance.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import answer_relevancy
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for AnswerRelevance. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "rag_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -92,12 +104,6 @@ def AnswerRelevance(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import answer_relevancy
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/ragas/AnswerSimilarity.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import answer_similarity
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for AnswerSimilarity. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -78,12 +90,6 @@ def AnswerSimilarity(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import answer_similarity
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/ragas/AspectCritique.py

@@ -8,9 +8,28 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import AspectCritic
+    from ragas.metrics._aspect_critic import (
+        coherence,
+        conciseness,
+        correctness,
+        harmfulness,
+        maliciousness,
+    )
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for AspectCritique. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 LOWER_IS_BETTER_ASPECTS = ["harmfulness", "maliciousness"]
 
 
@@ -101,20 +120,7 @@ def AspectCritique(
     )
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics.critique import AspectCritique as _AspectCritique
-        from ragas.metrics.critique import (
-            coherence,
-            conciseness,
-            correctness,
-            harmfulness,
-            maliciousness,
-        )
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
-    aspect_map = {
+    built_in_aspects = {
         "coherence": coherence,
         "conciseness": conciseness,
         "correctness": correctness,
@@ -136,16 +142,15 @@ def AspectCritique(
 
     df = get_renamed_columns(dataset._df, required_columns)
 
-    built_in_aspects = [aspect_map[aspect] for aspect in aspects]
     custom_aspects = (
         [
-            _AspectCritique(name=name, definition=description)
+            AspectCritic(name=name, definition=description)
             for name, description in additional_aspects
         ]
         if additional_aspects
         else []
     )
-    all_aspects = [*built_in_aspects, *custom_aspects]
+    all_aspects = [built_in_aspects[aspect] for aspect in aspects] + custom_aspects
 
     result_df = evaluate(
         Dataset.from_pandas(df), metrics=all_aspects, **get_ragas_config()

validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import context_entity_recall
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for ContextEntityRecall. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -84,12 +96,6 @@ def ContextEntityRecall(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import context_entity_recall
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import context_precision
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for ContextPrecision. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
@@ -79,12 +91,6 @@ def ContextPrecision(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import context_precision
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/ragas/ContextRecall.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import context_recall
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for ContextRecall. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
@@ -79,12 +91,6 @@ def ContextRecall(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import context_recall
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/ragas/ContextUtilization.py

@@ -0,0 +1,161 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import warnings
+
+import plotly.express as px
+from datasets import Dataset
+
+from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
+
+from .utils import get_ragas_config, get_renamed_columns
+
+try:
+    from ragas import evaluate
+    from ragas.metrics import context_utilization
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for ContextUtilization. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
+
+@tags("ragas", "llm", "retrieval_performance")
+@tasks("text_qa", "text_generation", "text_summarization", "text_classification")
+def ContextUtilization(
+    dataset,
+    question_column: str = "question",
+    contexts_column: str = "contexts",
+    answer_column: str = "answer",
+):  # noqa: B950
+    """
+    Assesses how effectively relevant context chunks are utilized in generating answers by evaluating their ranking
+    within the provided contexts.
+
+    ### Purpose
+
+    The Context Utilization test evaluates whether all of the answer-relevant items present in the contexts are ranked
+    higher within the provided retrieval results. This metric is essential for assessing the performance of models,
+    especially those involved in tasks such as text QA, text generation, text summarization, and text classification.
+
+    ### Test Mechanism
+
+    The test calculates Context Utilization using the formula:
+
+    $$
+    \\text{Context Utilization@K} = \\frac{\\sum_{k=1}^{K} \\left( \\text{Precision@k} \\times v_k \\right)}{\\text{Total number of relevant items in the top } K \\text{ results}}
+    $$
+    $$
+    \\text{Precision@k} = {\\text{true positives@k} \\over  (\\text{true positives@k} + \\text{false positives@k})}
+    $$
+
+    Where $K$ is the total number of chunks in `contexts` and $v_k \\in \\{0, 1\\}$ is the relevance indicator at rank $k$.
+
+
+    This test uses columns for questions, contexts, and answers from the dataset and computes context utilization
+    scores, generating a histogram and box plot for visualization.
+
+    #### Configuring Columns
+
+    This metric requires the following columns in your dataset:
+
+    - `question` (str): The text query that was input into the model.
+    - `contexts` (List[str]): A list of text contexts which are retrieved and which will be evaluated to
+       make sure they contain relevant info in the correct order.
+    - `answer` (str): The llm-generated response for the input `question`.
+
+    If the above data is not in the appropriate column, you can specify different column
+    names for these fields using the parameters `question_column`, `contexts_column`
+    and `ground_truth_column`.
+
+    For example, if your dataset has this data stored in different columns, you can
+    pass the following parameters:
+    ```python
+    {
+        "question_column": "question",
+        "contexts_column": "context_info"
+        "ground_truth_column": "my_ground_truth_col",
+    }
+    ```
+
+    If the data is stored as a dictionary in another column, specify the column and key
+    like this:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "contexts_column": f"{pred_col}.contexts",
+        "ground_truth_column": "my_ground_truth_col",
+    }
+    ```
+
+    For more complex situations, you can use a function to extract the data:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "contexts_column": lambda x: [x[pred_col]["context_message"]],
+        "ground_truth_column": "my_ground_truth_col",
+    }
+    ```
+
+    ### Signs of High Risk
+
+    - Very low mean or median context utilization scores, indicating poor usage of retrieved contexts.
+    - High standard deviation, suggesting inconsistent model performance.
+    - Low or minimal max scores, pointing to the model's failure to rank relevant contexts at top positions.
+
+    ### Strengths
+
+    - Quantifies the rank of relevant context chunks in generating responses.
+    - Provides clear visualizations through histograms and box plots for ease of interpretation.
+    - Adapts to different dataset schema by allowing configurable column names.
+
+    ### Limitations
+
+    - Assumes the relevance of context chunks is binary and may not capture nuances of partial relevance.
+    - Requires proper context retrieval to be effective; irrelevant context chunks can skew the results.
+    - Dependent on large sample sizes to provide stable and reliable estimates of utilization performance.
+    """
+    warnings.filterwarnings(
+        "ignore",
+        category=FutureWarning,
+        message="promote has been superseded by promote_options='default'.",
+    )
+
+    required_columns = {
+        "question": question_column,
+        "contexts": contexts_column,
+        "answer": answer_column,
+    }
+
+    df = get_renamed_columns(dataset._df, required_columns)
+
+    result_df = evaluate(
+        Dataset.from_pandas(df), metrics=[context_utilization], **get_ragas_config()
+    ).to_pandas()
+
+    fig_histogram = px.histogram(x=result_df["context_utilization"].to_list(), nbins=10)
+    fig_box = px.box(x=result_df["context_utilization"].to_list())
+
+    return (
+        {
+            # "Scores (will not be uploaded to UI)": result_df[
+            #     ["question", "contexts", "answer", "context_utilization"]
+            # ],
+            "Aggregate Scores": [
+                {
+                    "Mean Score": result_df["context_utilization"].mean(),
+                    "Median Score": result_df["context_utilization"].median(),
+                    "Max Score": result_df["context_utilization"].max(),
+                    "Min Score": result_df["context_utilization"].min(),
+                    "Standard Deviation": result_df["context_utilization"].std(),
+                    "Count": result_df.shape[0],
+                }
+            ],
+        },
+        fig_histogram,
+        fig_box,
+    )

validmind/tests/model_validation/ragas/Faithfulness.py

@@ -8,9 +8,21 @@ import plotly.express as px
 from datasets import Dataset
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
 
+try:
+    from ragas import evaluate
+    from ragas.metrics import faithfulness
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for Faithfulness. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
 
 @tags("ragas", "llm", "rag_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
@@ -78,12 +90,6 @@ def Faithfulness(
     }
     ```
     """
-    try:
-        from ragas import evaluate
-        from ragas.metrics import faithfulness
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

validmind/tests/model_validation/ragas/NoiseSensitivity.py

@@ -0,0 +1,158 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import warnings
+
+import plotly.express as px
+from datasets import Dataset
+
+from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
+
+from .utils import get_ragas_config, get_renamed_columns
+
+try:
+    from ragas import evaluate
+    from ragas.metrics import noise_sensitivity_relevant
+except ImportError as e:
+    raise MissingDependencyError(
+        "Missing required package `ragas` for NoiseSensitivity. "
+        "Please run `pip install validmind[llm]` to use LLM tests",
+        required_dependencies=["ragas"],
+        extra="llm",
+    ) from e
+
+
+@tags("ragas", "llm", "rag_performance")
+@tasks("text_qa", "text_generation", "text_summarization")
+def NoiseSensitivity(
+    dataset,
+    answer_column="answer",
+    contexts_column="contexts",
+    ground_truth_column="ground_truth",
+):
+    """
+    Assesses the sensitivity of a Large Language Model (LLM) to noise in retrieved context by measuring how often it
+    generates incorrect responses.
+
+    ### Purpose
+
+    The Noise Sensitivity test aims to measure how sensitive an LLM is to irrelevant or noisy information within the
+    contextual data used to generate its responses. A lower noise sensitivity score suggests better model robustness in
+    generating accurate answers from given contexts.
+
+    ### Test Mechanism
+
+    This test evaluates the model's answers by comparing the claims made in the generated response against the ground
+    truth and the retrieved context. The noise sensitivity score is calculated as:
+
+    $$
+    \\text{noise sensitivity} = {|\\text{Number of incorrect claims in answer}| \\over |\\text{Number of total claims in answer}|}
+    $$
+
+    The formula computes the fraction of incorrect claims to the total claims in the answer, using a dataset where
+    'answer', 'context', and 'ground_truth' columns are specified.
+
+    #### Configuring Columns
+
+    This metric requires the following columns in your dataset:
+
+    - `contexts` (List[str]): A list of text contexts which are retrieved to generate
+    the answer.
+    - `answer` (str): The response generated by the model
+    - `ground_truth` (str): The "correct" answer to the question
+
+    If the above data is not in the appropriate column, you can specify different column
+    names for these fields using the parameters `contexts_column` and `answer_column`.
+
+    For example, if your dataset has this data stored in different columns, you can
+    pass the following parameters:
+    ```python
+    {
+        "contexts_column": "context_info"
+        "answer_column": "my_answer_col",
+    }
+    ```
+
+    If the data is stored as a dictionary in another column, specify the column and key
+    like this:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "contexts_column": f"{pred_col}.contexts",
+        "answer_column": f"{pred_col}.answer",
+    }
+    ```
+
+    For more complex situations, you can use a function to extract the data:
+    ```python
+    pred_col = dataset.prediction_column(model)
+    params = {
+        "contexts_column": lambda row: [row[pred_col]["context_message"]],
+        "answer_column": lambda row: "\\n\\n".join(row[pred_col]["messages"]),
+    }
+
+    ### Signs of High Risk
+
+    - High noise sensitivity scores across multiple samples.
+    - Significant deviation between mean and median noise sensitivity scores.
+    - High standard deviation indicating inconsistency in the model's performance.
+
+    ### Strengths
+
+    - Provides a quantitative measure of how well the LLM handles noisy or irrelevant context.
+    - Easy integration and configuration using column parameters.
+    - Utilizes both histogram and box plot visualizations to analyze score distribution.
+
+    ### Limitations
+
+    - Requires accurate ground truth that aligns with the generated answers.
+    - Assumes the context provided is sufficiently granular to assess noise sensitivity.
+    - Primarily applicable to tasks like text QA, text generation, and text summarization where contextual relevance is
+    critical.
+    """
+    warnings.filterwarnings(
+        "ignore",
+        category=FutureWarning,
+        message="promote has been superseded by promote_options='default'.",
+    )
+
+    required_columns = {
+        "answer": answer_column,
+        "contexts": contexts_column,
+        "ground_truth": ground_truth_column,
+    }
+
+    df = get_renamed_columns(dataset._df, required_columns)
+
+    result_df = evaluate(
+        Dataset.from_pandas(df),
+        metrics=[noise_sensitivity_relevant],
+        **get_ragas_config(),
+    ).to_pandas()
+
+    fig_histogram = px.histogram(
+        x=result_df["noise_sensitivity_relevant"].to_list(), nbins=10
+    )
+    fig_box = px.box(x=result_df["noise_sensitivity_relevant"].to_list())
+
+    return (
+        {
+            # "Scores (will not be uploaded to UI)": result_df[
+            #     ["contexts", "answer", "ground_truth", "noise_sensitivity_relevant"]
+            # ],
+            "Aggregate Scores": [
+                {
+                    "Mean Score": result_df["noise_sensitivity_relevant"].mean(),
+                    "Median Score": result_df["noise_sensitivity_relevant"].median(),
+                    "Max Score": result_df["noise_sensitivity_relevant"].max(),
+                    "Min Score": result_df["noise_sensitivity_relevant"].min(),
+                    "Standard Deviation": result_df["noise_sensitivity_relevant"].std(),
+                    "Count": result_df.shape[0],
+                }
+            ],
+        },
+        fig_histogram,
+        fig_box,
+    )

validmind/tests/model_validation/sklearn/FeatureImportance.py

@@ -81,9 +81,9 @@ def FeatureImportance(dataset, model, num_features=3):
     # Dynamically add feature columns to the result
     for i in range(num_features):
         if i < len(top_features):
-            result[f"Feature {i + 1}"] = (
-                f"[{top_features[i][0]}; {top_features[i][1]:.4f}]"
-            )
+            result[
+                f"Feature {i + 1}"
+            ] = f"[{top_features[i][0]}; {top_features[i][1]:.4f}]"
         else:
             result[f"Feature {i + 1}"] = None
 

validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py

@@ -109,7 +109,7 @@ class PermutationFeatureImportance(Metric):
             )
         )
         fig.update_layout(
-            title_text="Permutation Importances (train set)",
+            title_text="Permutation Importances",
             yaxis=dict(
                 tickmode="linear",  # set tick mode to linear
                 dtick=1,  # set interval between ticks

validmind/tests/model_validation/sklearn/RegressionR2Square.py

@@ -3,11 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import pandas as pd
-
 from sklearn import metrics
 
-from validmind.tests.model_validation.statsmodels.statsutils import adj_r2_score
 from validmind import tags, tasks
+from validmind.tests.model_validation.statsmodels.statsutils import adj_r2_score
 
 
 @tags("sklearn", "model_performance")