validmind 2.1.0__py3-none-any.whl → 2.2.2__py3-none-any.whl

validmind/tests/data_validation/nlp/PolarityAndSubjectivity.py

@@ -0,0 +1,48 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+
+import pandas as pd
+import plotly.express as px
+from textblob import TextBlob
+
+from validmind import tags, tasks
+
+
+@tags("data_validation")
+@tasks("nlp")
+def PolarityAndSubjectivity(dataset):
+    """
+    Analyzes the polarity and subjectivity of text data within a dataset.
+
+    This method processes a dataset containing textual data to compute the polarity and
+    subjectivity scores using TextBlob, and returns a Plotly scatter plot visualizing
+    these scores.
+
+    Args:
+        dataset (Dataset): A dataset object which must have a `df` attribute (a pandas DataFrame)
+            and a `text_column` attribute indicating the name of the column containing text.
+
+    Returns:
+        plotly.graph_objs._figure.Figure: A Plotly scatter plot of polarity vs subjectivity.
+    """
+    # Function to calculate sentiment and subjectivity
+    def analyze_sentiment(text):
+        analysis = TextBlob(text)
+        return analysis.sentiment.polarity, analysis.sentiment.subjectivity
+
+    data = pd.DataFrame()
+    # Apply the function to each row
+    data[["polarity", "subjectivity"]] = dataset.df[dataset.text_column].apply(
+        lambda x: pd.Series(analyze_sentiment(x))
+    )
+
+    # Create a Plotly scatter plot
+    fig = px.scatter(
+        data, x="polarity", y="subjectivity", title="Polarity vs Subjectivity"
+    )
+    fig.update_traces(textposition="top center")
+    fig.update_layout(xaxis_title="Polarity", yaxis_title="Subjectivity")
+
+    return fig

validmind/tests/data_validation/nlp/Punctuations.py

@@ -72,25 +72,24 @@ class Punctuations(Metric):
         text_column = self.inputs.dataset.text_column
         corpus = create_corpus(self.inputs.dataset.df, text_column=text_column)
 
-        dic = defaultdict(int)
         special = string.punctuation
+        dic = defaultdict(int, {key: 0 for key in special})
         for i in corpus:
             if i in special:
                 dic[i] += 1
-
+        figures = []
+        # if dic:
         fig = plt.figure()
         x, y = zip(*dic.items())
         plt.bar(x, y, color="#17C37B")
-
+        figures.append(
+            Figure(
+                for_object=self,
+                key=self.key,
+                figure=fig,
+            )
+        )
         # Do this if you want to prevent the figure from being displayed
         plt.close("all")
 
-        return self.cache_results(
-            figures=[
-                Figure(
-                    for_object=self,
-                    key=self.key,
-                    figure=fig,
-                )
-            ]
-        )
+        return self.cache_results(figures=figures)

validmind/tests/data_validation/nlp/Sentiment.py

@@ -0,0 +1,57 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+
+import matplotlib.pyplot as plt
+import nltk
+import seaborn as sns
+from nltk.sentiment import SentimentIntensityAnalyzer
+
+from validmind import tags, tasks
+
+
+@tags("data_validation")
+@tasks("nlp")
+def Sentiment(dataset):
+    """
+    Analyzes the sentiment of text data within a dataset using the VADER sentiment analysis tool.
+
+    This method initializes the VADER SentimentIntensityAnalyzer and applies it to each text entry
+    in the specified column of the dataset's dataframe. It returns a KDE plot visualizing the distribution
+    of sentiment scores across the dataset.
+
+    Args:
+        dataset (Dataset): A dataset object which must have a `df` attribute (a pandas DataFrame)
+            and a `text_column` attribute indicating the name of the column containing text.
+
+    Returns:
+        matplotlib.figure.Figure: A KDE plot visualizing the distribution of sentiment scores.
+    """
+    nltk.download("vader_lexicon", quiet=True)
+    # Initialize VADER
+    sia = SentimentIntensityAnalyzer()
+
+    # Function to get VADER sentiment scores
+    def get_vader_sentiment(text):
+        sentiment_score = sia.polarity_scores(text)
+        return sentiment_score["compound"]
+
+    # Apply the function to each row
+    vader_sentiment = dataset.df[dataset.text_column].apply(get_vader_sentiment)
+
+    fig = plt.figure()
+    ax = sns.kdeplot(
+        x=vader_sentiment,
+        fill=True,
+        common_norm=False,
+        palette="crest",
+        alpha=0.5,
+        linewidth=0,
+    )
+    ax.set_title(f"Sentiment score of {dataset.text_column} ")
+    ax.set_xlabel("Sentiment score")
+
+    plt.close("all")
+
+    return fig

validmind/tests/data_validation/nlp/Toxicity.py

@@ -0,0 +1,45 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import evaluate
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+from validmind import tags, tasks
+
+
+@tags("data_validation")
+@tasks("nlp")
+def Toxicity(dataset):
+    """
+    Analyzes the toxicity of text data within a dataset using a pre-trained toxicity model.
+
+    This method loads a toxicity evaluation model and applies it to each text entry
+    in the specified column of the dataset's dataframe. It returns a KDE plot visualizing the distribution
+    of toxicity scores across the dataset.
+
+    Args:
+        dataset (Dataset): A dataset object which must have a `df` attribute (a pandas DataFrame)
+            and a `text_column` attribute indicating the name of the column containing text.
+
+    Returns:
+        matplotlib.figure.Figure: A KDE plot visualizing the distribution of toxicity scores.
+    """
+    toxicity = evaluate.load("toxicity")
+    input_text = dataset.df[dataset.text_column]
+    toxicity_scores = toxicity.compute(predictions=list(input_text.values))["toxicity"]
+
+    fig = plt.figure()
+    ax = sns.kdeplot(
+        x=toxicity_scores,
+        fill=True,
+        common_norm=False,
+        palette="crest",
+        alpha=0.5,
+        linewidth=0,
+    )
+    ax.set_title(f"Toxicity score of {dataset.text_column} ")
+    ax.set_xlabel("Toxicity score")
+    plt.close("all")
+    return fig

validmind/tests/decorator.py

@@ -153,7 +153,7 @@ def _get_run_method(func, inputs, params):
             test_id=self.test_id,
             description=inspect.getdoc(self),
             output_template=self.output_template,
-            inputs=list(inputs.keys()),
+            inputs=self.get_accessed_inputs(),
         )
 
         return self.result
@@ -264,7 +264,7 @@ def metric(func_or_id):
             {
                 "run": _get_run_method(func, inputs, params),
                 "required_inputs": list(inputs.keys()),
-                "default_parameters": params,
+                "default_params": {k: v["default"] for k, v in params.items()},
                 "__doc__": description,
                 "metadata": {
                     "task_types": tasks,

validmind/tests/model_validation/BertScore.py

@@ -2,116 +2,118 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-import itertools
-from dataclasses import dataclass
-
 import evaluate
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
 
 
-@dataclass
-class BertScore(Metric):
+@tags("nlp", "text_data", "visualization")
+@tasks("text_classification", "text_summarization")
+def BertScore(dataset, model):
     """
-    Evaluates text generation models' performance by calculating precision, recall, and F1 score based on BERT
+    Evaluates the quality of machine-generated text using BERTScore metrics and visualizes the results through histograms
+    and bar charts, alongside compiling a comprehensive table of descriptive statistics for each BERTScore metric.
+
+    **Purpose:**
+    This function is designed to assess the quality of text generated by machine learning models using BERTScore metrics.
+    BERTScore evaluates text generation models' performance by calculating precision, recall, and F1 score based on BERT
     contextual embeddings.
 
-    **Purpose**: The BERTScore metric is deployed to evaluate the competence of text generation models by focusing on
-    the similarity between the reference and the generated text. It employs the contextual embeddings from BERT models
-    to assess the similarity of the contents. This measures the extent to which a model has learned and can generate
-    contextually relevant results.
-
-    **Test Mechanism**: The true values derived from the model's test dataset and the model's predictions are employed
-    in this metric. BERTScore calculates the precision, recall, and F1 score of the model considering the contextual
-    similarity between the reference and the produced text. These scores are computed for each token in the predicted
-    sentences as compared to the reference sentences, while considering the cosine similarity with BERT embeddings. A
-    line plot depicting the score changes across row indexes is generated for each metric i.e., Precision, Recall, and
-    F1 Score.
-
-    **Signs of High Risk**:
-    - Observable downward trend in Precision, Recall, or F1 Score.
-    - Noticeable instability or fluctuation in these metrics. Lower Precision implies that predictions often
-    incorporate irrelevant contexts.
-    - Declining Recall suggests that the model frequently omits relevant contexts during predictions.
-    - Lower F1 score signals poor overall performance in both precision and recall.
-
-    **Strengths**:
-    - BERTScore efficiently detects the quality of text that requires to comprehend the context, a common requirement
-    in natural language processing tasks.
-    - This metric advances beyond the simple n-gram matching and considers the semantic similarity in the context,
-    thereby providing more meaningful evaluation results.
-    - The integrated visualization function allows tracking of the performance trends across different prediction sets.
-
-    **Limitations**:
-    - Dependence on BERT model embeddings for BERTScore implies that if the base BERT model is not suitable for a
-    specific task, it might impair the accuracy of BERTScore.
-    - Despite being good at understanding semantics, it might be incapable of capturing certain nuances in text
-    similarity that other metrics like BLEU or ROUGE could detect.
-    - Can be computationally expensive due to the utilization of BERT embeddings.
+    **Test Mechanism:**
+    The function starts by extracting the true and predicted values from the provided dataset and model. It then initializes
+    the BERTScore evaluator. For each pair of true and predicted texts, the function calculates the BERTScore metrics and
+    compiles them into a dataframe. Histograms and bar charts are generated for each BERTScore metric (Precision, Recall,
+    and F1 Score) to visualize their distribution. Additionally, a table of descriptive statistics (mean, median, standard
+    deviation, minimum, and maximum) is compiled for each metric, providing a comprehensive summary of the model's performance.
+
+    **Signs of High Risk:**
+    - Consistently low scores across BERTScore metrics could indicate poor quality in the generated text, suggesting that the model
+      fails to capture the essential content of the reference texts.
+    - Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
+    - Low recall scores may indicate that important information from the reference text is being omitted.
+    - An imbalanced performance between precision and recall, reflected by a low F1 Score, could signal issues in the model's ability
+      to balance informativeness and conciseness.
+
+    **Strengths:**
+    - Provides a multifaceted evaluation of text quality through different BERTScore metrics, offering a detailed view of model performance.
+    - Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the scores.
+    - Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
+
+    **Limitations:**
+    - BERTScore relies on the contextual embeddings from BERT models, which may not fully capture all nuances of text similarity.
+    - The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
+    - While useful for comparison, BERTScore metrics alone do not provide a complete assessment of a model's performance and should be
+      supplemented with other metrics and qualitative analysis.
     """
 
-    name = "bert_score"
-    required_inputs = ["model", "dataset"]
-
-    def run(self):
-        y_true = list(itertools.chain.from_iterable(self.inputs.dataset.y))
-        y_pred = self.inputs.dataset.y_pred(self.inputs.model)
-
-        # Load the bert evaluation metric
-        bert = evaluate.load("bertscore")
-
-        # Compute the BLEU score
-        bert_s = bert.compute(
-            predictions=y_pred,
-            references=y_true,
-            lang="en",
+    # Extract true and predicted values
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Ensure y_true and y_pred have the same length
+    if len(y_true) != len(y_pred):
+        min_length = min(len(y_true), len(y_pred))
+        y_true = y_true[:min_length]
+        y_pred = y_pred[:min_length]
+
+    # Load the BERT evaluation metric
+    bert = evaluate.load("bertscore")
+
+    # Compute the BERT score
+    bert_s = bert.compute(
+        predictions=y_pred,
+        references=y_true,
+        lang="en",
+    )
+
+    # Convert scores to a dataframe
+    metrics_df = pd.DataFrame(bert_s)
+    figures = []
+
+    # Generate histograms and bar charts for each score type
+    score_types = ["precision", "recall", "f1"]
+    score_names = ["Precision", "Recall", "F1 Score"]
+
+    for score_type, score_name in zip(score_types, score_names):
+        # Histogram
+        hist_fig = go.Figure(data=[go.Histogram(x=metrics_df[score_type])])
+        hist_fig.update_layout(
+            title=f"{score_name} Histogram",
+            xaxis_title=score_name,
+            yaxis_title="Count",
         )
+        figures.append(hist_fig)
 
-        metrics_df = pd.DataFrame(bert_s)
-        figures = []
-
-        # Visualization part
-        fig = go.Figure()
-
-        # Adding the line plots
-        fig.add_trace(
-            go.Scatter(
-                x=metrics_df.index,
-                y=metrics_df["precision"],
-                mode="lines+markers",
-                name="Precision",
-            )
-        )
-        fig.add_trace(
-            go.Scatter(
-                x=metrics_df.index,
-                y=metrics_df["recall"],
-                mode="lines+markers",
-                name="Recall",
-            )
-        )
-        fig.add_trace(
-            go.Scatter(
-                x=metrics_df.index,
-                y=metrics_df["f1"],
-                mode="lines+markers",
-                name="F1 Score",
-            )
-        )
-
-        fig.update_layout(
-            title="Bert Scores for Each Row",
+        # Bar Chart
+        bar_fig = go.Figure(data=[go.Bar(x=metrics_df.index, y=metrics_df[score_type])])
+        bar_fig.update_layout(
+            title=f"{score_name} Bar Chart",
             xaxis_title="Row Index",
-            yaxis_title="Score",
-        )
-        figures.append(
-            Figure(
-                for_object=self,
-                key=self.key,
-                figure=fig,
-            )
+            yaxis_title=score_name,
         )
-
-        return self.cache_results(figures=figures)
+        figures.append(bar_fig)
+
+    # Calculate statistics for each score type
+    stats_df = metrics_df.describe().loc[["mean", "50%", "max", "min", "std"]]
+    stats_df = stats_df.rename(
+        index={
+            "mean": "Mean Score",
+            "50%": "Median Score",
+            "max": "Max Score",
+            "min": "Min Score",
+            "std": "Standard Deviation",
+        }
+    ).T
+    stats_df["Count"] = len(metrics_df)
+
+    # Rename metrics for clarity
+    stats_df.index = stats_df.index.map(
+        {"precision": "Precision", "recall": "Recall", "f1": "F1 Score"}
+    )
+
+    # Create a DataFrame from all collected statistics
+    result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
+
+    return (result_df, *tuple(figures))

validmind/tests/model_validation/BleuScore.py

@@ -2,77 +2,106 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import evaluate
+import pandas as pd
+import plotly.graph_objects as go
 
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
+from validmind import tags, tasks
 
 
-@dataclass
-class BleuScore(Metric):
+@tags("nlp", "text_data", "visualization")
+@tasks("text_classification", "text_summarization")
+def BleuScore(dataset, model):
     """
-    Assesses translation quality by comparing machine-translated sentences with human-translated ones using BLEU score.
-
-    **Purpose**: The Bilingual Evaluation Understudy (BLEU) metric measures the quality of machine-translated text by
-    comparing it to human-translated text. This comparison is done at the sentence level and is designed to bring
-    machine translations closer to the quality of a professional human translation. It is commonly used in the field of
-    translation evaluation, and its purpose is to assess the accuracy of a model's output against that of a benchmark.
-
-    **Test Mechanism**: The BLEU score is implemented using the NLTK's word_tokenize function to split the text into
-    individual words. After tokenization, the evaluate library's BLEU metric calculates the BLEU score for each
-    translated sentence by comparing the model's translations (predictions) against the actual, correct translations
-    (references). The test algorithm then combines these individual scores into a single score that represents the
-    average 'distance' between the generated translations and the human translations across the entire test set.
-
-    **Signs of High Risk**:
-    - Low BLEU scores suggest high model risk. This could indicate significant discrepancies between the machine
-    translation and its human equivalent.
-    - This could be due to ineffective model learning, overfitting of training data, or inadequate handling of the
-    language's nuances.
-    - Machine biases toward a certain language style or translation mode can result in lower scores.
-
-    **Strengths**:
-    - The BLEU score's primary strength lies in its simplicity and interpretability. It offers a straightforward way to
-    assess translated text quality, and its calculations often align with human judgments.
-    - The BLEU score breaks down its evaluations at the sentence level, offering granular insights into any errors.
-    - The score consolidates the model’s performance into a single, comprehensive score, making it easy to compare and
-    monitor.
-
-    **Limitations**:
-    - The BLEU score heavily favours exact matches, which can create a bias towards literal translations. Thus, it may
-    fail to fully evaluate more complex or flexible translations that shy away from a word-for-word structure.
-    - The score does not directly measure the intelligibility or grammatical correctness of the translations.
-    - It may miss errors originating from subtle nuances in language, cultural contexts, or ambiguities.
+    Evaluates the quality of machine-generated text using BLEU metrics and visualizes the results through histograms
+    and bar charts, alongside compiling a comprehensive table of descriptive statistics for BLEU scores.
+
+    **Purpose:**
+    This function is designed to assess the quality of text generated by machine learning models using the BLEU metric.
+    BLEU, which stands for Bilingual Evaluation Understudy, is a metric used to evaluate the overlap of n-grams between
+    the machine-generated text and reference texts. This evaluation is crucial for tasks such as text summarization,
+    machine translation, and text generation, where the goal is to produce text that accurately reflects the content
+    and meaning of human-crafted references.
+
+    **Test Mechanism:**
+    The function starts by extracting the true and predicted values from the provided dataset and model. It then initializes
+    the BLEU evaluator. For each pair of true and predicted texts, the function calculates the BLEU scores and compiles them
+    into a dataframe. Histograms and bar charts are generated for the BLEU scores to visualize their distribution. Additionally,
+    a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum) is compiled for the BLEU scores,
+    providing a comprehensive summary of the model's performance.
+
+    **Signs of High Risk:**
+    - Consistently low BLEU scores could indicate poor quality in the generated text, suggesting that the model fails to capture
+      the essential content of the reference texts.
+    - Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
+    - Low recall scores may indicate that important information from the reference text is being omitted.
+    - An imbalanced performance between precision and recall, reflected by a low BLEU score, could signal issues in the model's
+      ability to balance informativeness and conciseness.
+
+    **Strengths:**
+    - Provides a straightforward and widely-used evaluation of text quality through BLEU scores.
+    - Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the scores.
+    - Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
+
+    **Limitations:**
+    - BLEU metrics primarily focus on n-gram overlap and may not fully capture semantic coherence, fluency, or grammatical quality
+      of the text.
+    - The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
+    - While useful for comparison, BLEU scores alone do not provide a complete assessment of a model's performance and should be
+      supplemented with other metrics and qualitative analysis.
     """
 
-    name = "bleu_score"
-    required_inputs = ["model", "dataset"]
+    # Extract true and predicted values
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
 
-    def run(self):
-        # Load the BLEU evaluation metric
-        bleu = evaluate.load("bleu")
+    # Load the BLEU evaluation metric
+    bleu = evaluate.load("bleu")
 
+    # Calculate BLEU scores
+    score_list = []
+    for y_t, y_p in zip(y_true, y_pred):
         # Compute the BLEU score
-        bleu = bleu.compute(
-            predictions=self.inputs.dataset.y_pred(self.inputs.model),
-            references=self.inputs.dataset.y,
-        )
-        return self.cache_results(metric_value={"blue_score_metric": bleu})
-
-    def summary(self, metric_value):
-        """
-        Build one table for summarizing the bleu score results
-        """
-        summary_bleu_score = metric_value["blue_score_metric"]
-
-        table = []
-        table.append(summary_bleu_score)
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=table,
-                    metadata=ResultTableMetadata(title="Bleu score Results"),
-                ),
-            ]
-        )
+        score = bleu.compute(predictions=[y_p], references=[[y_t]])
+        score_list.append(score["bleu"])
+
+    # Convert scores to a dataframe
+    metrics_df = pd.DataFrame(score_list, columns=["BLEU Score"])
+
+    figures = []
+
+    # Histogram for BLEU Score
+    hist_fig = go.Figure(data=[go.Histogram(x=metrics_df["BLEU Score"])])
+    hist_fig.update_layout(
+        title="BLEU Score Histogram",
+        xaxis_title="BLEU Score",
+        yaxis_title="Count",
+    )
+    figures.append(hist_fig)
+
+    # Bar Chart for BLEU Score
+    bar_fig = go.Figure(data=[go.Bar(x=metrics_df.index, y=metrics_df["BLEU Score"])])
+    bar_fig.update_layout(
+        title="BLEU Score Bar Chart",
+        xaxis_title="Row Index",
+        yaxis_title="BLEU Score",
+    )
+    figures.append(bar_fig)
+
+    # Calculate statistics for BLEU Score
+    stats_df = metrics_df.describe().loc[["mean", "50%", "max", "min", "std"]]
+    stats_df = stats_df.rename(
+        index={
+            "mean": "Mean Score",
+            "50%": "Median Score",
+            "max": "Max Score",
+            "min": "Min Score",
+            "std": "Standard Deviation",
+        }
+    ).T
+    stats_df["Count"] = len(metrics_df)
+
+    # Create a DataFrame from all collected statistics
+    result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
+
+    return (result_df, *tuple(figures))