validmind 2.8.27__py3-none-any.whl → 2.8.29__py3-none-any.whl

validmind/tests/model_validation/embeddings/CosineSimilarityHeatmap.py

@@ -2,24 +2,28 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import numpy as np
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.metrics.pairwise import cosine_similarity
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
 @tasks("text_qa", "text_generation", "text_summarization")
 def CosineSimilarityHeatmap(
-    dataset,
-    model,
+    dataset: VMDataset,
+    model: VMModel,
     title="Cosine Similarity Matrix",
     color="Cosine Similarity",
     xaxis_title="Index",
     yaxis_title="Index",
     color_scale="Blues",
-):
+) -> Tuple[go.Figure, RawData]:
     """
     Generates an interactive heatmap to visualize the cosine similarities among embeddings derived from a given model.
 

validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py

@@ -2,8 +2,11 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import numpy as np
 import plotly.express as px
+import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
@@ -11,7 +14,9 @@ from validmind.vm_models import VMDataset, VMModel
 
 @tags("llm", "text_data", "embeddings", "visualization")
 @tasks("feature_extraction")
-def DescriptiveAnalytics(dataset: VMDataset, model: VMModel):
+def DescriptiveAnalytics(
+    dataset: VMDataset, model: VMModel
+) -> Tuple[go.Figure, go.Figure, go.Figure, RawData]:
     """
     Evaluates statistical properties of text embeddings in an ML model via mean, median, and standard deviation
     histograms.

validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py

@@ -2,9 +2,10 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from typing import Union
+from typing import Tuple, Union
 
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.manifold import TSNE
 
 from validmind import RawData, tags, tasks
@@ -17,11 +18,11 @@ logger = get_logger(__name__)
 @tags("llm", "text_data", "embeddings", "visualization")
 @tasks("feature_extraction")
 def EmbeddingsVisualization2D(
-    model: VMModel,
     dataset: VMDataset,
+    model: VMModel,
     cluster_column: Union[str, None] = None,
     perplexity: int = 30,
-):
+) -> Tuple[go.Figure, RawData]:
     """
     Visualizes 2D representation of text embeddings generated by a model using t-SNE technique.
 

validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py

@@ -3,18 +3,23 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 from itertools import combinations
+from typing import List, Tuple
 
 import numpy as np
 import pandas as pd
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.metrics.pairwise import euclidean_distances
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
 @tasks("text_qa", "text_generation", "text_summarization")
-def EuclideanDistanceComparison(dataset, models):
+def EuclideanDistanceComparison(
+    dataset: VMDataset, models: List[VMModel]
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Assesses and visualizes the dissimilarity between model embeddings using Euclidean distance, providing insights
     into model behavior and potential redundancy or diversity.

validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py

@@ -2,24 +2,28 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import numpy as np
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.metrics.pairwise import euclidean_distances
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
 @tasks("text_qa", "text_generation", "text_summarization")
 def EuclideanDistanceHeatmap(
-    dataset,
-    model,
+    dataset: VMDataset,
+    model: VMModel,
     title="Euclidean Distance Matrix",
     color="Euclidean Distance",
     xaxis_title="Index",
     yaxis_title="Index",
     color_scale="Blues",
-):
+) -> Tuple[go.Figure, RawData]:
     """
     Generates an interactive heatmap to visualize the Euclidean distances among embeddings derived from a given model.
 

validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py

@@ -3,19 +3,24 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import itertools
+from typing import Tuple
 
 import numpy as np
 import pandas as pd
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.decomposition import PCA
 from sklearn.preprocessing import StandardScaler
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
 @tasks("text_qa", "text_generation", "text_summarization")
-def PCAComponentsPairwisePlots(dataset, model, n_components=3):
+def PCAComponentsPairwisePlots(
+    dataset: VMDataset, model: VMModel, n_components: int = 3
+) -> Tuple[go.Figure, RawData]:
     """
     Generates scatter plots for pairwise combinations of principal component analysis (PCA) components of model
     embeddings.

validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py

@@ -3,7 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import re
-from typing import Dict
+from typing import Dict, Tuple
+
+import pandas as pd
+import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
@@ -18,7 +21,7 @@ def StabilityAnalysisKeyword(
     model: VMModel,
     keyword_dict: Dict[str, str],
     mean_similarity_threshold: float = 0.7,
-):
+) -> Tuple[go.Figure, pd.DataFrame, RawData]:
     """
     Evaluates robustness of embedding models to keyword swaps in the test dataset.
 

validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py

@@ -4,6 +4,10 @@
 
 import random
 import string
+from typing import Tuple
+
+import pandas as pd
+import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
@@ -69,7 +73,7 @@ def StabilityAnalysisRandomNoise(
     model: VMModel,
     probability: float = 0.02,
     mean_similarity_threshold: float = 0.7,
-):
+) -> Tuple[go.Figure, pd.DataFrame, RawData]:
     """
     Assesses the robustness of text embeddings models to random noise introduced via text perturbations.
 

validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py

@@ -3,8 +3,11 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import random
+from typing import Tuple
 
 import nltk
+import pandas as pd
+import plotly.graph_objects as go
 from nltk.corpus import wordnet as wn
 
 from validmind import RawData, tags, tasks
@@ -20,7 +23,7 @@ def StabilityAnalysisSynonyms(
     model: VMModel,
     probability: float = 0.02,
     mean_similarity_threshold: float = 0.7,
-):
+) -> Tuple[go.Figure, pd.DataFrame, RawData]:
     """
     Evaluates the stability of text embeddings models when words in test data are replaced by their synonyms randomly.
 

validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py

@@ -2,6 +2,10 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
+import pandas as pd
+import plotly.graph_objects as go
 from transformers import MarianMTModel, MarianTokenizer
 
 from validmind import RawData, tags, tasks
@@ -21,7 +25,7 @@ def StabilityAnalysisTranslation(
     source_lang: str = "en",
     target_lang: str = "fr",
     mean_similarity_threshold: float = 0.7,
-):
+) -> Tuple[go.Figure, pd.DataFrame, RawData]:
     """
     Evaluates robustness of text embeddings models to noise introduced by translating the original text to another
     language and back.

validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py

@@ -3,25 +3,28 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import itertools
+from typing import Tuple
 
 import numpy as np
 import pandas as pd
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.manifold import TSNE
 from sklearn.preprocessing import StandardScaler
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
 @tasks("text_qa", "text_generation", "text_summarization")
 def TSNEComponentsPairwisePlots(
-    dataset,
-    model,
-    n_components=2,
-    perplexity=30,
-    title="t-SNE",
-):
+    dataset: VMDataset,
+    model: VMModel,
+    n_components: int = 2,
+    perplexity: int = 30,
+    title: str = "t-SNE",
+) -> Tuple[go.Figure, RawData]:
     """
     Creates scatter plots for pairwise combinations of t-SNE components to visualize embeddings and highlight potential
     clustering structures.

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, Tuple
 
 import plotly.express as px
+import plotly.graph_objects as go
 from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
+from validmind.vm_models import VMDataset
 
 from .utils import get_ragas_config, get_renamed_columns
 
@@ -30,13 +33,13 @@ except ImportError as e:
 @tags("ragas", "llm")
 @tasks("text_qa", "text_generation", "text_summarization")
 def AnswerCorrectness(
-    dataset,
-    user_input_column="user_input",
-    response_column="response",
-    reference_column="reference",
+    dataset: VMDataset,
+    user_input_column: str = "user_input",
+    response_column: str = "response",
+    reference_column: str = "reference",
     judge_llm=None,
     judge_embeddings=None,
-):
+) -> Tuple[Dict[str, list], go.Figure, go.Figure, RawData]:
     """
     Evaluates the correctness of answers in a dataset with respect to the provided ground
     truths and visualizes the results in a histogram.

validmind/tests/model_validation/ragas/AspectCritic.py

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, List, Optional, Tuple
 
 import plotly.express as px
+import plotly.graph_objects as go
 from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
+from validmind.vm_models import VMDataset
 
 from .utils import get_ragas_config, get_renamed_columns
 
@@ -39,21 +42,21 @@ LOWER_IS_BETTER_ASPECTS = ["harmfulness", "maliciousness"]
 @tags("ragas", "llm", "qualitative")
 @tasks("text_summarization", "text_generation", "text_qa")
 def AspectCritic(
-    dataset,
-    user_input_column="user_input",
-    response_column="response",
-    retrieved_contexts_column=None,
-    aspects: list = [
+    dataset: VMDataset,
+    user_input_column: str = "user_input",
+    response_column: str = "response",
+    retrieved_contexts_column: Optional[str] = None,
+    aspects: List[str] = [
         "coherence",
         "conciseness",
         "correctness",
         "harmfulness",
         "maliciousness",
     ],
-    additional_aspects: list = None,
+    additional_aspects: Optional[List[Tuple[str, str]]] = None,
     judge_llm=None,
     judge_embeddings=None,
-):
+) -> Tuple[Dict[str, list], go.Figure, RawData]:
     """
     Evaluates generations against the following aspects: harmfulness, maliciousness,
     coherence, correctness, and conciseness.
@@ -146,8 +149,8 @@ def AspectCritic(
 
     if retrieved_contexts_column:
         required_columns["retrieved_contexts"] = retrieved_contexts_column
-
     df = get_renamed_columns(dataset._df, required_columns)
+    df = df[required_columns.keys()]
 
     custom_aspects = (
         [

validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, Tuple
 
 import plotly.express as px
+import plotly.graph_objects as go
 from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
+from validmind.vm_models import VMDataset
 
 from .utils import get_ragas_config, get_renamed_columns
 
@@ -30,12 +33,12 @@ except ImportError as e:
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
 def ContextEntityRecall(
-    dataset,
+    dataset: VMDataset,
     retrieved_contexts_column: str = "retrieved_contexts",
     reference_column: str = "reference",
     judge_llm=None,
     judge_embeddings=None,
-):
+) -> Tuple[Dict[str, list], go.Figure, go.Figure, RawData]:
     """
     Evaluates the context entity recall for dataset entries and visualizes the results.
 

validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, Tuple
 
 import plotly.express as px
+import plotly.graph_objects as go
 from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
+from validmind.vm_models import VMDataset
 
 from .utils import get_ragas_config, get_renamed_columns
 
@@ -30,13 +33,13 @@ except ImportError as e:
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
 def ContextPrecision(
-    dataset,
+    dataset: VMDataset,
     user_input_column: str = "user_input",
     retrieved_contexts_column: str = "retrieved_contexts",
     reference_column: str = "reference",
     judge_llm=None,
     judge_embeddings=None,
-):  # noqa: B950
+) -> Tuple[Dict[str, list], go.Figure, go.Figure, RawData]:
     """
     Context Precision is a metric that evaluates whether all of the ground-truth
     relevant items present in the contexts are ranked higher or not. Ideally all the

validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, Tuple
 
 import plotly.express as px
+import plotly.graph_objects as go
 from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
+from validmind.vm_models import VMDataset
 
 from .utils import get_ragas_config, get_renamed_columns
 
@@ -30,13 +33,13 @@ except ImportError as e:
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
 def ContextPrecisionWithoutReference(
-    dataset,
+    dataset: VMDataset,
     user_input_column: str = "user_input",
     retrieved_contexts_column: str = "retrieved_contexts",
     response_column: str = "response",
     judge_llm=None,
     judge_embeddings=None,
-):  # noqa: B950
+) -> Tuple[Dict[str, list], go.Figure, go.Figure, RawData]:
     """
     Context Precision Without Reference is a metric used to evaluate the relevance of
     retrieved contexts compared to the expected response for a given user input. This

validmind/tests/model_validation/ragas/ContextRecall.py

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, Tuple
 
 import plotly.express as px
+import plotly.graph_objects as go
 from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
+from validmind.vm_models import VMDataset
 
 from .utils import get_ragas_config, get_renamed_columns
 
@@ -30,13 +33,13 @@ except ImportError as e:
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
 def ContextRecall(
-    dataset,
+    dataset: VMDataset,
     user_input_column: str = "user_input",
     retrieved_contexts_column: str = "retrieved_contexts",
     reference_column: str = "reference",
     judge_llm=None,
     judge_embeddings=None,
-):
+) -> Tuple[Dict[str, list], go.Figure, go.Figure, RawData]:
     """
     Context recall measures the extent to which the retrieved context aligns with the
     annotated answer, treated as the ground truth. It is computed based on the `ground
@@ -109,6 +112,7 @@ def ContextRecall(
     }
 
     df = get_renamed_columns(dataset._df, required_columns)
+    df = df[required_columns.keys()]
 
     result_df = evaluate(
         Dataset.from_pandas(df),

validmind/tests/model_validation/ragas/Faithfulness.py

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, Tuple
 
 import plotly.express as px
+import plotly.graph_objects as go
 from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
+from validmind.vm_models import VMDataset
 
 from .utils import get_ragas_config, get_renamed_columns
 
@@ -30,13 +33,13 @@ except ImportError as e:
 @tags("ragas", "llm", "rag_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
 def Faithfulness(
-    dataset,
-    user_input_column="user_input",
-    response_column="response",
-    retrieved_contexts_column="retrieved_contexts",
+    dataset: VMDataset,
+    user_input_column: str = "user_input",
+    response_column: str = "response",
+    retrieved_contexts_column: str = "retrieved_contexts",
     judge_llm=None,
     judge_embeddings=None,
-):  # noqa
+) -> Tuple[Dict[str, list], go.Figure, go.Figure, RawData]:
     """
     Evaluates the faithfulness of the generated answers with respect to retrieved contexts.
 
@@ -115,6 +118,7 @@ def Faithfulness(
 
     df = get_renamed_columns(dataset._df, required_columns)
 
+    df = df[required_columns.keys()]
     result_df = evaluate(
         Dataset.from_pandas(df),
         metrics=[faithfulness()],

validmind/tests/model_validation/ragas/NoiseSensitivity.py

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, Tuple
 
 import plotly.express as px
+import plotly.graph_objects as go
 from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
+from validmind.vm_models import VMDataset
 
 from .utils import get_ragas_config, get_renamed_columns
 
@@ -32,15 +35,15 @@ VALID_FOCUS_VALUES = ["relevant", "irrelevant"]
 @tags("ragas", "llm", "rag_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
 def NoiseSensitivity(
-    dataset,
-    response_column="response",
-    retrieved_contexts_column="retrieved_contexts",
-    reference_column="reference",
-    focus="relevant",
-    user_input_column="user_input",
+    dataset: VMDataset,
+    response_column: str = "response",
+    retrieved_contexts_column: str = "retrieved_contexts",
+    reference_column: str = "reference",
+    focus: str = "relevant",
+    user_input_column: str = "user_input",
     judge_llm=None,
     judge_embeddings=None,
-):
+) -> Tuple[Dict[str, list], go.Figure, go.Figure, RawData]:
     """
     Assesses the sensitivity of a Large Language Model (LLM) to noise in retrieved context by measuring how often it
     generates incorrect responses.

validmind/tests/model_validation/ragas/ResponseRelevancy.py

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, Tuple
 
 import plotly.express as px
+import plotly.graph_objects as go
 from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
+from validmind.vm_models import VMDataset
 
 from .utils import get_ragas_config, get_renamed_columns
 
@@ -30,13 +33,13 @@ except ImportError as e:
 @tags("ragas", "llm", "rag_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
 def ResponseRelevancy(
-    dataset,
-    user_input_column="user_input",
-    retrieved_contexts_column=None,
-    response_column="response",
+    dataset: VMDataset,
+    user_input_column: str = "user_input",
+    retrieved_contexts_column: str = None,
+    response_column: str = "response",
     judge_llm=None,
     judge_embeddings=None,
-):
+) -> Tuple[Dict[str, list], go.Figure, go.Figure, RawData]:
     """
     Assesses how pertinent the generated answer is to the given prompt.
 
@@ -124,6 +127,7 @@ def ResponseRelevancy(
         required_columns["retrieved_contexts"] = retrieved_contexts_column
 
     df = get_renamed_columns(dataset._df, required_columns)
+    df = df[required_columns.keys()]
 
     metrics = [response_relevancy()]
 
@@ -134,7 +138,6 @@ def ResponseRelevancy(
     ).to_pandas()
 
     score_column = "answer_relevancy"
-
     fig_histogram = px.histogram(
         x=result_df[score_column].to_list(), nbins=10, title="Response Relevancy"
     )