validmind 2.8.28__py3-none-any.whl → 2.9.1__py3-none-any.whl

validmind/tests/model_validation/TimeSeriesPredictionWithCI.py

@@ -2,17 +2,24 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 from scipy.stats import norm
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("model_predictions", "visualization")
 @tasks("regression", "time_series_forecasting")
-def TimeSeriesPredictionWithCI(dataset, model, confidence=0.95):
+def TimeSeriesPredictionWithCI(
+    dataset: VMDataset,
+    model: VMModel,
+    confidence: float = 0.95,
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Assesses predictive accuracy and uncertainty in time series models, highlighting breaches beyond confidence
     intervals.

validmind/tests/model_validation/TimeSeriesPredictionsPlot.py

@@ -2,14 +2,20 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("model_predictions", "visualization")
 @tasks("regression", "time_series_forecasting")
-def TimeSeriesPredictionsPlot(dataset, model):
+def TimeSeriesPredictionsPlot(
+    dataset: VMDataset,
+    model: VMModel,
+) -> Tuple[go.Figure, RawData]:
     """
     Plot actual vs predicted values for time series data and generate a visual comparison for the model.
 

validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py

@@ -2,17 +2,22 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Optional, Tuple
 
 import pandas as pd
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn import metrics
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("model_performance", "sklearn")
 @tasks("regression", "time_series_forecasting")
-def TimeSeriesR2SquareBySegments(dataset, model, segments=None):
+def TimeSeriesR2SquareBySegments(
+    dataset: VMDataset, model: VMModel, segments: Optional[int] = None
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Evaluates the R-Squared values of regression models over specified time segments in time series data to assess
     segment-wise model performance.

validmind/tests/model_validation/TokenDisparity.py

@@ -2,15 +2,20 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("nlp", "text_data", "visualization")
 @tasks("text_classification", "text_summarization")
-def TokenDisparity(dataset, model):
+def TokenDisparity(
+    dataset: VMDataset, model: VMModel
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Evaluates the token disparity between reference and generated texts, visualizing the results through histograms and
     bar charts, alongside compiling a comprehensive table of descriptive statistics for token counts.

validmind/tests/model_validation/ToxicityScore.py

@@ -2,16 +2,21 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import evaluate
 import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("nlp", "text_data", "visualization")
 @tasks("text_classification", "text_summarization")
-def ToxicityScore(dataset, model):
+def ToxicityScore(
+    dataset: VMDataset, model: VMModel
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Assesses the toxicity levels of texts generated by NLP models to identify and mitigate harmful or offensive content.
 

validmind/tests/model_validation/embeddings/ClusterDistribution.py

@@ -2,7 +2,10 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.cluster import KMeans
 
 from validmind import RawData, tags, tasks
@@ -11,7 +14,9 @@ from validmind.vm_models import VMDataset, VMModel
 
 @tags("llm", "text_data", "embeddings", "visualization")
 @tasks("feature_extraction")
-def ClusterDistribution(model: VMModel, dataset: VMDataset, num_clusters: int = 5):
+def ClusterDistribution(
+    model: VMModel, dataset: VMDataset, num_clusters: int = 5
+) -> Tuple[go.Figure, RawData]:
     """
     Assesses the distribution of text embeddings across clusters produced by a model using KMeans clustering.
 

validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py

@@ -3,18 +3,23 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 from itertools import combinations
+from typing import List, Tuple
 
 import numpy as np
 import pandas as pd
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.metrics.pairwise import cosine_similarity
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
 @tasks("text_qa", "text_generation", "text_summarization")
-def CosineSimilarityComparison(dataset, models):
+def CosineSimilarityComparison(
+    dataset: VMDataset, models: List[VMModel]
+) -> Tuple[go.Figure, RawData, pd.DataFrame]:
     """
     Assesses the similarity between embeddings generated by different models using Cosine Similarity, providing both
     statistical and visual insights.

validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py

@@ -2,7 +2,10 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.metrics.pairwise import cosine_similarity
 
 from validmind import RawData, tags, tasks
@@ -11,7 +14,9 @@ from validmind.vm_models import VMDataset, VMModel
 
 @tags("llm", "text_data", "embeddings", "visualization")
 @tasks("feature_extraction")
-def CosineSimilarityDistribution(dataset: VMDataset, model: VMModel):
+def CosineSimilarityDistribution(
+    dataset: VMDataset, model: VMModel
+) -> Tuple[go.Figure, RawData]:
     """
     Assesses the similarity between predicted text embeddings from a model using a Cosine Similarity distribution
     histogram.

validmind/tests/model_validation/embeddings/CosineSimilarityHeatmap.py

@@ -2,24 +2,28 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import numpy as np
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.metrics.pairwise import cosine_similarity
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
 @tasks("text_qa", "text_generation", "text_summarization")
 def CosineSimilarityHeatmap(
-    dataset,
-    model,
+    dataset: VMDataset,
+    model: VMModel,
     title="Cosine Similarity Matrix",
     color="Cosine Similarity",
     xaxis_title="Index",
     yaxis_title="Index",
     color_scale="Blues",
-):
+) -> Tuple[go.Figure, RawData]:
     """
     Generates an interactive heatmap to visualize the cosine similarities among embeddings derived from a given model.
 

validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py

@@ -2,8 +2,11 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import numpy as np
 import plotly.express as px
+import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
@@ -11,7 +14,9 @@ from validmind.vm_models import VMDataset, VMModel
 
 @tags("llm", "text_data", "embeddings", "visualization")
 @tasks("feature_extraction")
-def DescriptiveAnalytics(dataset: VMDataset, model: VMModel):
+def DescriptiveAnalytics(
+    dataset: VMDataset, model: VMModel
+) -> Tuple[go.Figure, go.Figure, go.Figure, RawData]:
     """
     Evaluates statistical properties of text embeddings in an ML model via mean, median, and standard deviation
     histograms.

validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py

@@ -2,9 +2,10 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from typing import Union
+from typing import Tuple, Union
 
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.manifold import TSNE
 
 from validmind import RawData, tags, tasks
@@ -17,11 +18,11 @@ logger = get_logger(__name__)
 @tags("llm", "text_data", "embeddings", "visualization")
 @tasks("feature_extraction")
 def EmbeddingsVisualization2D(
-    model: VMModel,
     dataset: VMDataset,
+    model: VMModel,
     cluster_column: Union[str, None] = None,
     perplexity: int = 30,
-):
+) -> Tuple[go.Figure, RawData]:
     """
     Visualizes 2D representation of text embeddings generated by a model using t-SNE technique.
 

validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py

@@ -3,18 +3,23 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 from itertools import combinations
+from typing import List, Tuple
 
 import numpy as np
 import pandas as pd
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.metrics.pairwise import euclidean_distances
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
 @tasks("text_qa", "text_generation", "text_summarization")
-def EuclideanDistanceComparison(dataset, models):
+def EuclideanDistanceComparison(
+    dataset: VMDataset, models: List[VMModel]
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Assesses and visualizes the dissimilarity between model embeddings using Euclidean distance, providing insights
     into model behavior and potential redundancy or diversity.

validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py

@@ -2,24 +2,28 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import numpy as np
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.metrics.pairwise import euclidean_distances
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
 @tasks("text_qa", "text_generation", "text_summarization")
 def EuclideanDistanceHeatmap(
-    dataset,
-    model,
+    dataset: VMDataset,
+    model: VMModel,
     title="Euclidean Distance Matrix",
     color="Euclidean Distance",
     xaxis_title="Index",
     yaxis_title="Index",
     color_scale="Blues",
-):
+) -> Tuple[go.Figure, RawData]:
     """
     Generates an interactive heatmap to visualize the Euclidean distances among embeddings derived from a given model.
 

validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py

@@ -3,19 +3,24 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import itertools
+from typing import Tuple
 
 import numpy as np
 import pandas as pd
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.decomposition import PCA
 from sklearn.preprocessing import StandardScaler
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
 @tasks("text_qa", "text_generation", "text_summarization")
-def PCAComponentsPairwisePlots(dataset, model, n_components=3):
+def PCAComponentsPairwisePlots(
+    dataset: VMDataset, model: VMModel, n_components: int = 3
+) -> Tuple[go.Figure, RawData]:
     """
     Generates scatter plots for pairwise combinations of principal component analysis (PCA) components of model
     embeddings.

validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py

@@ -3,7 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import re
-from typing import Dict
+from typing import Dict, Tuple
+
+import pandas as pd
+import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
@@ -18,7 +21,7 @@ def StabilityAnalysisKeyword(
     model: VMModel,
     keyword_dict: Dict[str, str],
     mean_similarity_threshold: float = 0.7,
-):
+) -> Tuple[go.Figure, pd.DataFrame, RawData]:
     """
     Evaluates robustness of embedding models to keyword swaps in the test dataset.
 

validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py

@@ -4,6 +4,10 @@
 
 import random
 import string
+from typing import Tuple
+
+import pandas as pd
+import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
@@ -69,7 +73,7 @@ def StabilityAnalysisRandomNoise(
     model: VMModel,
     probability: float = 0.02,
     mean_similarity_threshold: float = 0.7,
-):
+) -> Tuple[go.Figure, pd.DataFrame, RawData]:
     """
     Assesses the robustness of text embeddings models to random noise introduced via text perturbations.
 

validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py

@@ -3,8 +3,11 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import random
+from typing import Tuple
 
 import nltk
+import pandas as pd
+import plotly.graph_objects as go
 from nltk.corpus import wordnet as wn
 
 from validmind import RawData, tags, tasks
@@ -20,7 +23,7 @@ def StabilityAnalysisSynonyms(
     model: VMModel,
     probability: float = 0.02,
     mean_similarity_threshold: float = 0.7,
-):
+) -> Tuple[go.Figure, pd.DataFrame, RawData]:
     """
     Evaluates the stability of text embeddings models when words in test data are replaced by their synonyms randomly.
 

validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py

@@ -2,6 +2,10 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
+import pandas as pd
+import plotly.graph_objects as go
 from transformers import MarianMTModel, MarianTokenizer
 
 from validmind import RawData, tags, tasks
@@ -21,7 +25,7 @@ def StabilityAnalysisTranslation(
     source_lang: str = "en",
     target_lang: str = "fr",
     mean_similarity_threshold: float = 0.7,
-):
+) -> Tuple[go.Figure, pd.DataFrame, RawData]:
     """
     Evaluates robustness of text embeddings models to noise introduced by translating the original text to another
     language and back.

validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py

@@ -3,25 +3,28 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import itertools
+from typing import Tuple
 
 import numpy as np
 import pandas as pd
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.manifold import TSNE
 from sklearn.preprocessing import StandardScaler
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
 @tasks("text_qa", "text_generation", "text_summarization")
 def TSNEComponentsPairwisePlots(
-    dataset,
-    model,
-    n_components=2,
-    perplexity=30,
-    title="t-SNE",
-):
+    dataset: VMDataset,
+    model: VMModel,
+    n_components: int = 2,
+    perplexity: int = 30,
+    title: str = "t-SNE",
+) -> Tuple[go.Figure, RawData]:
     """
     Creates scatter plots for pairwise combinations of t-SNE components to visualize embeddings and highlight potential
     clustering structures.

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, Tuple
 
 import plotly.express as px
+import plotly.graph_objects as go
 from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
+from validmind.vm_models import VMDataset
 
 from .utils import get_ragas_config, get_renamed_columns
 
@@ -30,13 +33,13 @@ except ImportError as e:
 @tags("ragas", "llm")
 @tasks("text_qa", "text_generation", "text_summarization")
 def AnswerCorrectness(
-    dataset,
-    user_input_column="user_input",
-    response_column="response",
-    reference_column="reference",
+    dataset: VMDataset,
+    user_input_column: str = "user_input",
+    response_column: str = "response",
+    reference_column: str = "reference",
     judge_llm=None,
     judge_embeddings=None,
-):
+) -> Tuple[Dict[str, list], go.Figure, go.Figure, RawData]:
     """
     Evaluates the correctness of answers in a dataset with respect to the provided ground
     truths and visualizes the results in a histogram.

validmind/tests/model_validation/ragas/AspectCritic.py

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, List, Optional, Tuple
 
 import plotly.express as px
+import plotly.graph_objects as go
 from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
+from validmind.vm_models import VMDataset
 
 from .utils import get_ragas_config, get_renamed_columns
 
@@ -39,21 +42,21 @@ LOWER_IS_BETTER_ASPECTS = ["harmfulness", "maliciousness"]
 @tags("ragas", "llm", "qualitative")
 @tasks("text_summarization", "text_generation", "text_qa")
 def AspectCritic(
-    dataset,
-    user_input_column="user_input",
-    response_column="response",
-    retrieved_contexts_column=None,
-    aspects: list = [
+    dataset: VMDataset,
+    user_input_column: str = "user_input",
+    response_column: str = "response",
+    retrieved_contexts_column: Optional[str] = None,
+    aspects: List[str] = [
         "coherence",
         "conciseness",
         "correctness",
         "harmfulness",
         "maliciousness",
     ],
-    additional_aspects: list = None,
+    additional_aspects: Optional[List[Tuple[str, str]]] = None,
     judge_llm=None,
     judge_embeddings=None,
-):
+) -> Tuple[Dict[str, list], go.Figure, RawData]:
     """
     Evaluates generations against the following aspects: harmfulness, maliciousness,
     coherence, correctness, and conciseness.
@@ -146,8 +149,8 @@ def AspectCritic(
 
     if retrieved_contexts_column:
         required_columns["retrieved_contexts"] = retrieved_contexts_column
-
     df = get_renamed_columns(dataset._df, required_columns)
+    df = df[required_columns.keys()]
 
     custom_aspects = (
         [

validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, Tuple
 
 import plotly.express as px
+import plotly.graph_objects as go
 from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
+from validmind.vm_models import VMDataset
 
 from .utils import get_ragas_config, get_renamed_columns
 
@@ -30,12 +33,12 @@ except ImportError as e:
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization")
 def ContextEntityRecall(
-    dataset,
+    dataset: VMDataset,
     retrieved_contexts_column: str = "retrieved_contexts",
     reference_column: str = "reference",
     judge_llm=None,
     judge_embeddings=None,
-):
+) -> Tuple[Dict[str, list], go.Figure, go.Figure, RawData]:
     """
     Evaluates the context entity recall for dataset entries and visualizes the results.
 

validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, Tuple
 
 import plotly.express as px
+import plotly.graph_objects as go
 from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
+from validmind.vm_models import VMDataset
 
 from .utils import get_ragas_config, get_renamed_columns
 
@@ -30,13 +33,13 @@ except ImportError as e:
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
 def ContextPrecision(
-    dataset,
+    dataset: VMDataset,
     user_input_column: str = "user_input",
     retrieved_contexts_column: str = "retrieved_contexts",
     reference_column: str = "reference",
     judge_llm=None,
     judge_embeddings=None,
-):  # noqa: B950
+) -> Tuple[Dict[str, list], go.Figure, go.Figure, RawData]:
     """
     Context Precision is a metric that evaluates whether all of the ground-truth
     relevant items present in the contexts are ranked higher or not. Ideally all the

validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py

@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, Tuple
 
 import plotly.express as px
+import plotly.graph_objects as go
 from datasets import Dataset
 
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
+from validmind.vm_models import VMDataset
 
 from .utils import get_ragas_config, get_renamed_columns
 
@@ -30,13 +33,13 @@ except ImportError as e:
 @tags("ragas", "llm", "retrieval_performance")
 @tasks("text_qa", "text_generation", "text_summarization", "text_classification")
 def ContextPrecisionWithoutReference(
-    dataset,
+    dataset: VMDataset,
     user_input_column: str = "user_input",
     retrieved_contexts_column: str = "retrieved_contexts",
     response_column: str = "response",
     judge_llm=None,
     judge_embeddings=None,
-):  # noqa: B950
+) -> Tuple[Dict[str, list], go.Figure, go.Figure, RawData]:
     """
     Context Precision Without Reference is a metric used to evaluate the relevance of
     retrieved contexts compared to the expected response for a given user input. This