validmind 2.2.6__py3-none-any.whl → 2.3.1__py3-none-any.whl

validmind/tests/data_validation/TabularDateTimeHistograms.py

@@ -2,8 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-import matplotlib.pyplot as plt
-import seaborn as sns
+import pandas as pd
+import plotly.graph_objects as go
 
 from validmind.vm_models import Figure, Metric
 
@@ -50,45 +50,41 @@ class TabularDateTimeHistograms(Metric):
 
     metadata = {
         "task_types": ["classification", "regression"],
-        "tags": ["tabular_data", "visualization"],
+        "tags": ["time_series_data", "visualization"],
     }
 
     def run(self):
         df = self.inputs.dataset.df
 
-        # Extract datetime columns from the dataset
-        datetime_columns = df.select_dtypes(include=["datetime64"]).columns.tolist()
-
-        if len(datetime_columns) == 0:
-            raise ValueError("No datetime columns found in the dataset")
+        # Check if the index is a datetime type
+        if not isinstance(df.index, (pd.DatetimeIndex, pd.PeriodIndex)):
+            raise ValueError("Index must be a datetime type")
 
         figures = []
-        for col in datetime_columns:
-            plt.figure()
-            fig, _ = plt.subplots()
-
-            # Calculate the difference between consecutive dates and convert to days
-            date_diffs = df[col].sort_values().diff().dt.days.dropna()
-
-            # Filter out 0 values
-            date_diffs = date_diffs[date_diffs != 0]
-
-            ax = sns.histplot(date_diffs, kde=False, bins=30)
-            plt.title(f"{col}", weight="bold", fontsize=20)
-
-            plt.xticks(fontsize=18)
-            plt.yticks(fontsize=18)
-            ax.set_xlabel("Days Between Consecutive Dates", fontsize=18)
-            ax.set_ylabel("Frequency", fontsize=18)
-            figures.append(
-                Figure(
-                    for_object=self,
-                    key=f"{self.key}:{col}",
-                    figure=fig,
-                )
-            )
 
-        plt.close("all")
+        # Calculate the difference between consecutive dates in the index
+        date_diffs = df.index.to_series().sort_values().diff().dt.days.dropna()
+
+        # Filter out 0 values
+        date_diffs = date_diffs[date_diffs != 0]
+
+        # Create a histogram using Plotly
+        fig = go.Figure()
+        fig.add_trace(go.Histogram(x=date_diffs, nbinsx=30))
+        fig.update_layout(
+            title="Index",
+            xaxis_title="Days Between Consecutive Dates",
+            yaxis_title="Frequency",
+            font=dict(size=18),
+        )
+
+        figures.append(
+            Figure(
+                for_object=self,
+                key=f"{self.key}:index",
+                figure=fig,
+            )
+        )
 
         return self.cache_results(
             figures=figures,

validmind/tests/data_validation/WOEBinPlots.py

@@ -58,7 +58,7 @@ class WOEBinPlots(Metric):
     """
 
     name = "woe_bin_plots"
-    required_context = ["dataset"]
+    required_inputs = ["dataset"]
     default_params = {"breaks_adj": None, "fig_height": 600, "fig_width": 500}
     metadata = {
         "task_types": ["classification"],

validmind/tests/data_validation/WOEBinTable.py

@@ -46,7 +46,7 @@ class WOEBinTable(Metric):
     """
 
     name = "woe_bin_table"
-    required_context = ["dataset"]
+    required_inputs = ["dataset"]
     default_params = {"breaks_adj": None}
     metadata = {
         "task_types": ["classification"],

validmind/tests/{model_validation/statsmodels → data_validation}/ZivotAndrewsArch.py

@@ -4,9 +4,14 @@
 
 from dataclasses import dataclass
 
+import pandas as pd
 from arch.unitroot import ZivotAndrews
+from numpy.linalg import LinAlgError
 
-from validmind.vm_models import Metric
+from validmind.logging import get_logger
+from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
+
+logger = get_logger(__name__)
 
 
 @dataclass
@@ -57,14 +62,63 @@ class ZivotAndrewsArch(Metric):
         """
         dataset = self.inputs.dataset.df
 
-        za_values = {}
+        # Check if the dataset is a time series
+        if not isinstance(dataset.index, (pd.DatetimeIndex, pd.PeriodIndex)):
+            raise ValueError(
+                "Dataset index must be a datetime or period index for time series analysis."
+            )
+
+        # Preprocessing: Drop rows with any NaN values
+        if dataset.isnull().values.any():
+            logger.warning(
+                "Dataset contains missing values. Rows with NaNs will be dropped."
+            )
+            dataset = dataset.dropna()
+
+        # Convert to numeric and handle non-numeric data
+        dataset = dataset.apply(pd.to_numeric, errors="coerce")
+
+        # Initialize a list to store Zivot-Andrews results
+        za_values = []
+
         for col in dataset.columns:
-            za = ZivotAndrews(dataset[col].values)
-            za_values[col] = {
-                "stat": za.stat,
-                "pvalue": za.pvalue,
-                "usedlag": za.lags,
-                "nobs": za.nobs,
-            }
-
-        return self.cache_results(za_values)
+            try:
+                za = ZivotAndrews(dataset[col].values)
+                za_values.append(
+                    {
+                        "Variable": col,
+                        "stat": za.stat,
+                        "pvalue": za.pvalue,
+                        "usedlag": za.lags,
+                        "nobs": za.nobs,
+                    }
+                )
+            except (LinAlgError, ValueError) as e:
+                logger.error(f"Error while processing column '{col}'. Details: {e}")
+                za_values.append(
+                    {
+                        "Variable": col,
+                        "stat": None,
+                        "pvalue": None,
+                        "usedlag": None,
+                        "nobs": None,
+                        "error": str(e),
+                    }
+                )
+
+        return self.cache_results({"zivot_andrews_results": za_values})
+
+    def summary(self, metric_value):
+        """
+        Build a table for summarizing the Zivot-Andrews results
+        """
+        za_results = metric_value["zivot_andrews_results"]
+
+        return ResultSummary(
+            results=[
+                ResultTable(
+                    data=za_results,
+                    metadata=ResultTableMetadata(title="Zivot-Andrews Test Results"),
+                )
+            ]
+        )

validmind/tests/data_validation/nlp/CommonWords.py

@@ -52,7 +52,7 @@ class CommonWords(Metric):
     """
 
     name = "common_words"
-    required_inputs = ["dataset", "dataset.text_column"]
+    required_inputs = ["dataset"]
     metadata = {
         "task_types": ["text_classification", "text_summarization"],
         "tags": ["nlp", "text_data", "visualization", "frequency_analysis"],

validmind/tests/data_validation/nlp/Hashtags.py

@@ -54,7 +54,7 @@ class Hashtags(ThresholdTest):
     """
 
     name = "hashtags"
-    required_inputs = ["dataset", "dataset.text_column"]
+    required_inputs = ["dataset"]
     default_params = {"top_hashtags": 25}
     metadata = {
         "task_types": ["text_classification", "text_summarization"],

validmind/tests/data_validation/nlp/Mentions.py

@@ -54,7 +54,7 @@ class Mentions(ThresholdTest):
 
     name = "mentions"
 
-    required_inputs = ["dataset", "dataset.text_column"]
+    required_inputs = ["dataset"]
     default_params = {"top_mentions": 25}
     metadata = {
         "task_types": ["text_classification", "text_summarization"],

validmind/tests/data_validation/nlp/PolarityAndSubjectivity.py

@@ -10,7 +10,7 @@ from textblob import TextBlob
 from validmind import tags, tasks
 
 
-@tags("data_validation")
+@tags("nlp", "text_data", "data_validation")
 @tasks("nlp")
 def PolarityAndSubjectivity(dataset):
     """
@@ -27,6 +27,7 @@ def PolarityAndSubjectivity(dataset):
     Returns:
         plotly.graph_objs._figure.Figure: A Plotly scatter plot of polarity vs subjectivity.
     """
+
     # Function to calculate sentiment and subjectivity
     def analyze_sentiment(text):
         analysis = TextBlob(text)

validmind/tests/data_validation/nlp/Punctuations.py

@@ -51,7 +51,7 @@ class Punctuations(Metric):
     """
 
     name = "punctuations"
-    required_inputs = ["dataset", "dataset.text_column"]
+    required_inputs = ["dataset"]
     metadata = {
         "task_types": ["text_classification", "text_summarization"],
         "tags": ["nlp", "text_data", "visualization", "frequency_analysis"],

validmind/tests/data_validation/nlp/Sentiment.py

@@ -11,7 +11,7 @@ from nltk.sentiment import SentimentIntensityAnalyzer
 from validmind import tags, tasks
 
 
-@tags("data_validation")
+@tags("nlp", "text_data", "data_validation")
 @tasks("nlp")
 def Sentiment(dataset):
     """

validmind/tests/data_validation/nlp/TextDescription.py

@@ -60,7 +60,7 @@ class TextDescription(Metric):
     """
 
     name = "text_description"
-    required_inputs = ["dataset", "dataset.text_column"]
+    required_inputs = ["dataset"]
     default_params = {
         "unwanted_tokens": {
             "s",
@@ -79,6 +79,10 @@ class TextDescription(Metric):
         "num_top_words": 3,
         "lang": "english",
     }
+    metadata = {
+        "task_types": ["text_classification", "text_summarization"],
+        "tags": ["nlp", "text_data", "visualization"],
+    }
 
     def general_text_metrics(self, df, text_column):
         nltk.download("punkt", quiet=True)

validmind/tests/data_validation/nlp/Toxicity.py

@@ -9,7 +9,7 @@ import seaborn as sns
 from validmind import tags, tasks
 
 
-@tags("data_validation")
+@tags("nlp", "text_data", "data_validation")
 @tasks("nlp")
 def Toxicity(dataset):
     """

validmind/tests/decorator.py

@@ -13,9 +13,9 @@ from uuid import uuid4
 
 import pandas as pd
 
+from validmind.ai.test_descriptions import get_description_metadata
 from validmind.errors import MissingRequiredTestInputError
 from validmind.logging import get_logger
-from validmind.utils import get_description_metadata
 from validmind.vm_models import (
     Metric,
     MetricResult,

validmind/tests/model_validation/FeaturesAUC.py

@@ -55,10 +55,12 @@ class FeaturesAUC(Metric):
     }
 
     def run(self):
-        x = self.inputs.dataset.x_df()
-        y = self.inputs.dataset.y_df()
+        dataset = self.inputs.dataset
+        x = dataset.x_df()
+        y = dataset.y_df()
+        n_targets = dataset.df[dataset.target_column].nunique()
 
-        if y.nunique() != 2:
+        if n_targets != 2:
             raise SkipTestError("FeaturesAUC metric requires a binary target variable.")
 
         aucs = pd.DataFrame(index=x.columns, columns=["AUC"])

validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py

@@ -9,7 +9,11 @@ import pandas as pd
 import plotly.express as px
 from sklearn.metrics.pairwise import cosine_similarity
 
+from validmind import tags, tasks
 
+
+@tags("visualization", "dimensionality_reduction", "embeddings")
+@tasks("text_qa", "text_generation", "text_summarization")
 def CosineSimilarityComparison(dataset, models):
     """
     Computes pairwise cosine similarities between model embeddings and visualizes the results through bar charts,

validmind/tests/model_validation/embeddings/CosineSimilarityHeatmap.py

@@ -6,7 +6,11 @@ import numpy as np
 import plotly.express as px
 from sklearn.metrics.pairwise import cosine_similarity
 
+from validmind import tags, tasks
 
+
+@tags("visualization", "dimensionality_reduction", "embeddings")
+@tasks("text_qa", "text_generation", "text_summarization")
 def CosineSimilarityHeatmap(
     dataset,
     model,

validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py

@@ -9,7 +9,11 @@ import pandas as pd
 import plotly.express as px
 from sklearn.metrics.pairwise import euclidean_distances
 
+from validmind import tags, tasks
 
+
+@tags("visualization", "dimensionality_reduction", "embeddings")
+@tasks("text_qa", "text_generation", "text_summarization")
 def EuclideanDistanceComparison(dataset, models):
     """
     Computes pairwise Euclidean distances between model embeddings and visualizes the results through bar charts,

validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py

@@ -6,7 +6,11 @@ import numpy as np
 import plotly.express as px
 from sklearn.metrics.pairwise import euclidean_distances
 
+from validmind import tags, tasks
 
+
+@tags("visualization", "dimensionality_reduction", "embeddings")
+@tasks("text_qa", "text_generation", "text_summarization")
 def EuclideanDistanceHeatmap(
     dataset,
     model,

validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py

@@ -10,7 +10,11 @@ import plotly.express as px
 from sklearn.decomposition import PCA
 from sklearn.preprocessing import StandardScaler
 
+from validmind import tags, tasks
 
+
+@tags("visualization", "dimensionality_reduction", "embeddings")
+@tasks("text_qa", "text_generation", "text_summarization")
 def PCAComponentsPairwisePlots(dataset, model, n_components=3):
     """
     Generates scatter plots for pairwise combinations of principal component analysis (PCA) components of model embeddings.

validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py

@@ -10,7 +10,11 @@ import plotly.express as px
 from sklearn.manifold import TSNE
 from sklearn.preprocessing import StandardScaler
 
+from validmind import tags, tasks
 
+
+@tags("visualization", "dimensionality_reduction", "embeddings")
+@tasks("text_qa", "text_generation", "text_summarization")
 def TSNEComponentsPairwisePlots(
     dataset,
     model,

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -11,7 +11,7 @@ from ragas.metrics import answer_correctness
 
 from validmind import tags, tasks
 
-from .utils import get_renamed_columns
+from .utils import get_ragas_config, get_renamed_columns
 
 
 @tags("ragas", "llm")
@@ -104,7 +104,7 @@ def AnswerCorrectness(
     df = get_renamed_columns(dataset.df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[answer_correctness]
+        Dataset.from_pandas(df), metrics=[answer_correctness], **get_ragas_config()
     ).to_pandas()
 
     fig_histogram = px.histogram(x=result_df["answer_correctness"].to_list(), nbins=10)
@@ -112,7 +112,7 @@ def AnswerCorrectness(
 
     return (
         {
-            "Scores": result_df[
+            "Scores (will not be uploaded to UI)": result_df[
                 ["question", "answer", "ground_truth", "answer_correctness"]
             ],
             "Aggregate Scores": [

validmind/tests/model_validation/ragas/AnswerRelevance.py

@@ -11,7 +11,7 @@ from ragas.metrics import answer_relevancy
 
 from validmind import tags, tasks
 
-from .utils import get_renamed_columns
+from .utils import get_ragas_config, get_renamed_columns
 
 
 @tags("ragas", "llm", "rag_performance")
@@ -108,8 +108,7 @@ def AnswerRelevance(
     df = get_renamed_columns(dataset.df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df),
-        metrics=[answer_relevancy],
+        Dataset.from_pandas(df), metrics=[answer_relevancy], **get_ragas_config()
     ).to_pandas()
 
     fig_histogram = px.histogram(x=result_df["answer_relevancy"].to_list(), nbins=10)
@@ -117,7 +116,9 @@ def AnswerRelevance(
 
     return (
         {
-            "Scores": result_df[["question", "contexts", "answer", "answer_relevancy"]],
+            "Scores (will not be uploaded to UI)": result_df[
+                ["question", "contexts", "answer", "answer_relevancy"]
+            ],
             "Aggregate Scores": [
                 {
                     "Mean Score": result_df["answer_relevancy"].mean(),

validmind/tests/model_validation/ragas/AnswerSimilarity.py

@@ -11,7 +11,7 @@ from ragas.metrics import answer_similarity
 
 from validmind import tags, tasks
 
-from .utils import get_renamed_columns
+from .utils import get_ragas_config, get_renamed_columns
 
 
 @tags("ragas", "llm")
@@ -93,8 +93,7 @@ def AnswerSimilarity(
     df = get_renamed_columns(dataset.df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df),
-        metrics=[answer_similarity],
+        Dataset.from_pandas(df), metrics=[answer_similarity], **get_ragas_config()
     ).to_pandas()
 
     fig_histogram = px.histogram(x=result_df["answer_similarity"].to_list(), nbins=10)
@@ -102,7 +101,9 @@ def AnswerSimilarity(
 
     return (
         {
-            "Scores": result_df[["answer", "ground_truth", "answer_similarity"]],
+            "Scores (will not be uploaded to UI)": result_df[
+                ["answer", "ground_truth", "answer_similarity"]
+            ],
             "Aggregate Scores": [
                 {
                     "Mean Score": result_df["answer_similarity"].mean(),

validmind/tests/model_validation/ragas/AspectCritique.py

@@ -18,7 +18,7 @@ from ragas.metrics.critique import (
 
 from validmind import tags, tasks
 
-from .utils import get_renamed_columns
+from .utils import get_ragas_config, get_renamed_columns
 
 aspect_map = {
     "coherence": coherence,
@@ -36,14 +36,14 @@ def AspectCritique(
     question_column="question",
     answer_column="answer",
     contexts_column="contexts",
-    aspects: list = [
+    aspects: list = [  # noqa: B006 this is fine as immutable default since it never gets modified
         "coherence",
         "conciseness",
         "correctness",
         "harmfulness",
         "maliciousness",
     ],
-    additional_aspects: list = [],
+    additional_aspects: list = None,
 ):
     """
     Evaluates generations against the following aspects: harmfulness, maliciousness,
@@ -131,13 +131,19 @@ def AspectCritique(
     df = get_renamed_columns(dataset.df, required_columns)
 
     built_in_aspects = [aspect_map[aspect] for aspect in aspects]
-    custom_aspects = [
-        _AspectCritique(name=name, definition=description)
-        for name, description in additional_aspects
-    ]
+    custom_aspects = (
+        [
+            _AspectCritique(name=name, definition=description)
+            for name, description in additional_aspects
+        ]
+        if additional_aspects
+        else []
+    )
     all_aspects = [*built_in_aspects, *custom_aspects]
 
-    result_df = evaluate(Dataset.from_pandas(df), metrics=all_aspects).to_pandas()
+    result_df = evaluate(
+        Dataset.from_pandas(df), metrics=all_aspects, **get_ragas_config()
+    ).to_pandas()
 
     df_melted = result_df.melt(
         id_vars=["question", "answer", "contexts"],

validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -11,7 +11,7 @@ from ragas.metrics import context_entity_recall
 
 from validmind import tags, tasks
 
-from .utils import get_renamed_columns
+from .utils import get_ragas_config, get_renamed_columns
 
 
 @tags("ragas", "llm", "retrieval_performance")
@@ -99,8 +99,7 @@ def ContextEntityRecall(
     df = get_renamed_columns(dataset.df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df),
-        metrics=[context_entity_recall],
+        Dataset.from_pandas(df), metrics=[context_entity_recall], **get_ragas_config()
     ).to_pandas()
 
     fig_histogram = px.histogram(
@@ -110,7 +109,7 @@ def ContextEntityRecall(
 
     return (
         {
-            "Scores": result_df[
+            "Scores (will not be uploaded to UI)": result_df[
                 [
                     "contexts",
                     "ground_truth",

validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -11,7 +11,7 @@ from ragas.metrics import context_precision
 
 from validmind import tags, tasks
 
-from .utils import get_renamed_columns
+from .utils import get_ragas_config, get_renamed_columns
 
 
 @tags("ragas", "llm", "retrieval_performance")
@@ -21,7 +21,7 @@ def ContextPrecision(
     question_column: str = "question",
     contexts_column: str = "contexts",
     ground_truth_column: str = "ground_truth",
-):
+):  # noqa: B950
     """
     Context Precision is a metric that evaluates whether all of the ground-truth
     relevant items present in the contexts are ranked higher or not. Ideally all the
@@ -95,8 +95,7 @@ def ContextPrecision(
     df = get_renamed_columns(dataset.df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df),
-        metrics=[context_precision],
+        Dataset.from_pandas(df), metrics=[context_precision], **get_ragas_config()
     ).to_pandas()
 
     fig_histogram = px.histogram(x=result_df["context_precision"].to_list(), nbins=10)
@@ -104,7 +103,7 @@ def ContextPrecision(
 
     return (
         {
-            "Scores": result_df[
+            "Scores (will not be uploaded to UI)": result_df[
                 ["question", "contexts", "ground_truth", "context_precision"]
             ],
             "Aggregate Scores": [

validmind/tests/model_validation/ragas/ContextRecall.py

@@ -11,7 +11,7 @@ from ragas.metrics import context_recall
 
 from validmind import tags, tasks
 
-from .utils import get_renamed_columns
+from .utils import get_ragas_config, get_renamed_columns
 
 
 @tags("ragas", "llm", "retrieval_performance")
@@ -95,8 +95,7 @@ def ContextRecall(
     df = get_renamed_columns(dataset.df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df),
-        metrics=[context_recall],
+        Dataset.from_pandas(df), metrics=[context_recall], **get_ragas_config()
     ).to_pandas()
 
     fig_histogram = px.histogram(x=result_df["context_recall"].to_list(), nbins=10)
@@ -104,7 +103,7 @@ def ContextRecall(
 
     return (
         {
-            "Scores": result_df[
+            "Scores (will not be uploaded to UI)": result_df[
                 ["question", "contexts", "ground_truth", "context_recall"]
             ],
             "Aggregate Scores": [

validmind/tests/model_validation/ragas/ContextRelevancy.py

@@ -11,7 +11,7 @@ from ragas.metrics import context_relevancy
 
 from validmind import tags, tasks
 
-from .utils import get_renamed_columns
+from .utils import get_ragas_config, get_renamed_columns
 
 
 @tags("ragas", "llm", "retrieval_performance")
@@ -88,8 +88,7 @@ def ContextRelevancy(
     df = get_renamed_columns(dataset.df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df),
-        metrics=[context_relevancy],
+        Dataset.from_pandas(df), metrics=[context_relevancy], **get_ragas_config()
     ).to_pandas()
 
     fig_histogram = px.histogram(x=result_df["context_relevancy"].to_list(), nbins=10)
@@ -97,7 +96,9 @@ def ContextRelevancy(
 
     return (
         {
-            "Scores": result_df[["question", "contexts", "context_relevancy"]],
+            "Scores (will not be uploaded to UI)": result_df[
+                ["question", "contexts", "context_relevancy"]
+            ],
             "Aggregate Scores": [
                 {
                     "Mean Score": result_df["context_relevancy"].mean(),