validmind 2.1.1__py3-none-any.whl → 2.2.4__py3-none-any.whl

validmind/tests/model_validation/MeteorScore.py

@@ -2,91 +2,103 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import evaluate
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
 
 
-@dataclass
-class MeteorScore(Metric):
+@tags("nlp", "text_data", "visualization")
+@tasks("text_classification", "text_summarization")
+def MeteorScore(dataset, model):
     """
     Computes and visualizes the METEOR score for each text generation instance, assessing translation quality.
 
-    **Purpose**: METEOR (Metric for Evaluation of Translation with Explicit ORdering) is designed to evaluate the quality
-    of machine translations by comparing them against reference translations. It emphasizes both the accuracy and fluency
-    of translations, incorporating precision, recall, and word order into its assessment.
-
-    **Test Mechanism**: The METEOR score is computed for each pair of machine-generated translation (prediction) and its
-    corresponding human-produced reference. This is done by considering unigram matches between the translations, including
-    matches based on surface forms, stemmed forms, and synonyms. The score is a combination of unigram precision and recall,
-    adjusted for word order through a fragmentation penalty.
-
-    **Signs of High Risk**:
-    - Lower METEOR scores can indicate a lack of alignment between the machine-generated translations and their human-produced references, highlighting potential deficiencies in both the accuracy and fluency of translations.
-    - Significant discrepancies in word order or an excessive fragmentation penalty could signal issues with how the translation model processes and reconstructs sentence structures, potentially compromising the natural flow of translated text.
-    - Persistent underperformance across a variety of text types or linguistic contexts might suggest a broader inability of the model to adapt to the nuances of different languages or dialects, pointing towards gaps in its training or inherent limitations.
-
-    **Strengths**:
-    - Incorporates a balanced consideration of precision and recall, weighted towards recall to reflect the importance of
-      content coverage in translations.
+    **Purpose:**
+    METEOR (Metric for Evaluation of Translation with Explicit ORdering) is designed to evaluate the quality of machine translations
+    by comparing them against reference translations. It emphasizes both the accuracy and fluency of translations, incorporating
+    precision, recall, and word order into its assessment.
+
+    **Test Mechanism:**
+    The function starts by extracting the true and predicted values from the provided dataset and model. The METEOR score is computed
+    for each pair of machine-generated translation (prediction) and its corresponding human-produced reference. This is done by
+    considering unigram matches between the translations, including matches based on surface forms, stemmed forms, and synonyms.
+    The score is a combination of unigram precision and recall, adjusted for word order through a fragmentation penalty. Scores are
+    compiled into a dataframe, and histograms and bar charts are generated to visualize the distribution of METEOR scores. Additionally,
+    a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum) is compiled for the METEOR scores,
+    providing a comprehensive summary of the model's performance.
+
+    **Signs of High Risk:**
+    - Lower METEOR scores can indicate a lack of alignment between the machine-generated translations and their human-produced references,
+      highlighting potential deficiencies in both the accuracy and fluency of translations.
+    - Significant discrepancies in word order or an excessive fragmentation penalty could signal issues with how the translation model processes
+      and reconstructs sentence structures, potentially compromising the natural flow of translated text.
+    - Persistent underperformance across a variety of text types or linguistic contexts might suggest a broader inability of the model to adapt to the
+      nuances of different languages or dialects, pointing towards gaps in its training or inherent limitations.
+
+    **Strengths:**
+    - Incorporates a balanced consideration of precision and recall, weighted towards recall to reflect the importance of content coverage in translations.
     - Directly accounts for word order, offering a nuanced evaluation of translation fluency beyond simple lexical matching.
     - Adapts to various forms of lexical similarity, including synonyms and stemmed forms, allowing for flexible matching.
 
-    **Limitations**:
-    - While comprehensive, the complexity of METEOR's calculation can make it computationally intensive, especially for
-      large datasets.
-    - The use of external resources for synonym and stemming matching may introduce variability based on the resources'
-      quality and relevance to the specific translation task.
+    **Limitations:**
+    - While comprehensive, the complexity of METEOR's calculation can make it computationally intensive, especially for large datasets.
+    - The use of external resources for synonym and stemming matching may introduce variability based on the resources' quality and relevance to the specific
+      translation task.
     """
 
-    name = "meteor_score"
-    required_inputs = ["model", "dataset"]
-
-    def run(self):
-        # Load the METEOR metric
-        meteor = evaluate.load("meteor")
-
-        # Initialize a list to hold METEOR scores
-        meteor_scores = []
-
-        for prediction, reference in zip(
-            self.inputs.dataset.y_pred(self.inputs.model),
-            self.inputs.dataset.y,
-        ):
-            # Compute the METEOR score for the current prediction-reference pair
-            result = meteor.compute(predictions=[prediction], references=[reference])
-            meteor_scores.append(result["meteor"])
-
-        # Visualization of METEOR scores
-        figures = self.visualize_scores(meteor_scores)
-
-        return self.cache_results(figures=figures)
-
-    def visualize_scores(self, scores):
-        # Convert the scores list to a DataFrame for plotting
-        scores_df = pd.DataFrame(scores, columns=["METEOR Score"])
-
-        # Create a line plot of the METEOR scores
-        fig = go.Figure()
-        fig.add_trace(
-            go.Scatter(
-                x=scores_df.index,
-                y=scores_df["METEOR Score"],
-                mode="lines+markers",
-                name="METEOR Score",
-            )
-        )
-        fig.update_layout(
-            title="METEOR Scores Across Text Instances",
-            xaxis_title="Text Instance Index",
-            yaxis_title="METEOR Score",
-        )
-
-        # Wrap the Plotly figure for compatibility with your framework
-        figures = [Figure(for_object=self, key=self.key, figure=fig)]
-
-        return figures
+    # Extract true and predicted values
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Load the METEOR evaluation metric
+    meteor = evaluate.load("meteor")
+
+    # Calculate METEOR scores
+    score_list = []
+    for y_t, y_p in zip(y_true, y_pred):
+        # Compute the METEOR score
+        score = meteor.compute(predictions=[y_p], references=[y_t])
+        score_list.append(score["meteor"])
+
+    # Convert scores to a dataframe
+    metrics_df = pd.DataFrame(score_list, columns=["METEOR Score"])
+
+    figures = []
+
+    # Histogram for METEOR Score
+    hist_fig = go.Figure(data=[go.Histogram(x=metrics_df["METEOR Score"])])
+    hist_fig.update_layout(
+        title="METEOR Score Histogram",
+        xaxis_title="METEOR Score",
+        yaxis_title="Count",
+    )
+    figures.append(hist_fig)
+
+    # Bar Chart for METEOR Score
+    bar_fig = go.Figure(data=[go.Bar(x=metrics_df.index, y=metrics_df["METEOR Score"])])
+    bar_fig.update_layout(
+        title="METEOR Score Bar Chart",
+        xaxis_title="Row Index",
+        yaxis_title="METEOR Score",
+    )
+    figures.append(bar_fig)
+
+    # Calculate statistics for METEOR Score
+    stats_df = metrics_df.describe().loc[["mean", "50%", "max", "min", "std"]]
+    stats_df = stats_df.rename(
+        index={
+            "mean": "Mean Score",
+            "50%": "Median Score",
+            "max": "Max Score",
+            "min": "Min Score",
+            "std": "Standard Deviation",
+        }
+    ).T
+    stats_df["Count"] = len(metrics_df)
+
+    # Create a DataFrame from all collected statistics
+    result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
+
+    return (result_df, *tuple(figures))

validmind/tests/model_validation/RegardScore.py

@@ -2,142 +2,124 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-import itertools
-from dataclasses import dataclass
-
 import evaluate
+import pandas as pd
 import plotly.graph_objects as go
-from plotly.subplots import make_subplots
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
 
 
-@dataclass
-class RegardScore(Metric):
+@tags("nlp", "text_data", "visualization")
+@tasks("text_classification", "text_summarization")
+def RegardScore(dataset, model):
     """
+    Computes and visualizes the regard score for each text instance, assessing sentiment and potential biases.
+
     **Purpose:**
-    The `RegardScore` metric assesses the degree of regard—positive, negative, neutral, or other—present in the given text,
-    whether it's a classification or summarization result. Especially crucial for applications like sentiment analysis,
-    product reviews, or opinion mining, it provides a granular understanding of how the model perceives or generates content
-    in terms of favorability or sentiment.
+    The `RegardScore` metric is designed to evaluate the regard levels (positive, negative, neutral, or other) of texts generated by models. This helps in understanding the sentiment and biases in the generated content.
 
     **Test Mechanism:**
-    The metric ingests data primarily from the model's test dataset, extracting the input text, target text (true regard),
-    and the model's predicted regard. These elements undergo a series of consistency checks before being processed. Using
-    the `evaluate.load("regard")` tool, regard scores are computed for each segment of text. The results are then visualized
-    in a multi-subplot line graph, where each subplot corresponds to a particular category of regard (e.g., positive, negative,
-    neutral, other) against the input, target, and predicted texts.
+    The function starts by extracting the true and predicted values from the provided dataset and model. The regard scores are computed for each text using a preloaded `regard` evaluation tool. The scores are compiled into dataframes, and histograms and bar charts are generated to visualize the distribution of regard scores. Additionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum) is compiled for the regard scores, providing a comprehensive summary of the model's performance.
 
     **Signs of High Risk:**
-    Disparities between the target regard scores and the predicted regard scores may signify potential flaws or biases in
-    the model. For instance, if neutral inputs are consistently perceived as strongly positive or negative, this could
-    indicate the model's inability to correctly identify or generate balanced sentiments.
+    - Noticeable skewness in the histogram, especially when comparing the predicted regard scores with the target regard scores, could indicate biases or inconsistencies in the model.
+    - Lack of neutral scores in the model's predictions, despite a balanced distribution in the target data, might signal an issue.
 
     **Strengths:**
-    The metric's visual presentation, using line plots, provides an intuitive way to comprehend the model's regard assessment
-    across different text samples and regard categories. The color-coded lines associated with each regard category further
-    enhance the clarity of the comparison, making it simpler for stakeholders or researchers to infer the model's performance.
+    - Provides a clear evaluation of regard levels in generated texts, helping to ensure content appropriateness.
+    - Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of regard scores.
+    - Descriptive statistics offer a concise summary of the model's performance in generating texts with balanced sentiments.
 
     **Limitations:**
-    The `RegardScoreHistogram` metric emphasizes regard scores but may not always grasp intricate nuances or the true context
-    of texts. Its reliance on underlying tools, which might be trained on potentially biased datasets, can result in skewed
-    interpretations. Additionally, while the metric segments regard into discrete categories such as "positive" and "negative,"
-    real-world sentiments often exist on a more complex spectrum. The metric's efficacy is intertwined with the accuracy of
-    the model's predictions; any inherent model inaccuracies can impact the metric's reflection of true sentiments.
+    - The accuracy of the regard scores is contingent upon the underlying `regard` tool.
+    - The scores provide a broad overview but do not specify which portions or tokens of the text are responsible for high regard.
+    - Supplementary, in-depth analysis might be needed for granular insights.
     """
 
-    name = "regard_score"
-    required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["regard_score"],
-    }
-
-    def _get_datasets(self):
-        if not hasattr(self, "model"):
-            raise AttributeError("The 'model' attribute is missing.")
-
-        y_true = list(itertools.chain.from_iterable(self.inputs.dataset.y))
-        y_pred = self.inputs.dataset.y_pred(self.inputs.model)
-
-        if not len(y_true) == len(y_pred):
-            raise ValueError(
-                "Inconsistent lengths among input text, true summaries, and predicted summaries."
+    # Extract true and predicted values
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Load the regard evaluation metric
+    regard_tool = evaluate.load("regard")
+
+    # Function to calculate regard scores
+    def compute_regard_scores(texts):
+        scores = regard_tool.compute(data=texts)["regard"]
+        regard_dicts = [
+            dict((x["label"], x["score"]) for x in sublist) for sublist in scores
+        ]
+        return regard_dicts
+
+    # Calculate regard scores for true and predicted texts
+    true_regard = compute_regard_scores(y_true)
+    pred_regard = compute_regard_scores(y_pred)
+
+    # Convert scores to dataframes
+    true_df = pd.DataFrame(true_regard)
+    pred_df = pd.DataFrame(pred_regard)
+
+    figures = []
+
+    # Function to create histogram and bar chart for regard scores
+    def create_figures(df, title):
+        for category in df.columns:
+            # Histogram
+            hist_fig = go.Figure(data=[go.Histogram(x=df[category])])
+            hist_fig.update_layout(
+                title=f"{title} - {category.capitalize()} Histogram",
+                xaxis_title=category.capitalize(),
+                yaxis_title="Count",
             )
-
-        return y_true, y_pred
-
-    def regard_line_plot(self):
-        regard_tool = evaluate.load("regard")
-        y_true, y_pred = self._get_datasets()
-
-        dataframes = {
-            "Target Text": y_true,
-            "Predicted Summaries": y_pred,
-        }
-
-        total_text_columns = len(dataframes)
-        total_rows = total_text_columns * 2
-
-        categories_order = ["positive", "negative", "neutral", "other"]
-        category_colors = {
-            "negative": "#d9534f",
-            "neutral": "#5bc0de",
-            "other": "#f0ad4e",
-            "positive": "#5cb85c",
-        }
-
-        fig = make_subplots(
-            rows=total_rows,
-            cols=2,
-            subplot_titles=[
-                f"{col_name} {cat}"
-                for col_name in dataframes
-                for cat in categories_order
-            ],
-            shared_yaxes=True,
-            vertical_spacing=0.1,
-        )
-
-        row_offset = 0
-        for column_name, column_data in dataframes.items():
-            results = regard_tool.compute(data=column_data)["regard"]
-            regard_dicts = [
-                dict((x["label"], x["score"]) for x in sublist) for sublist in results
-            ]
-
-            for idx, category in enumerate(categories_order, start=1):
-                row, col = ((idx - 1) // 2 + 1 + row_offset, (idx - 1) % 2 + 1)
-                scores = [res_dict[category] for res_dict in regard_dicts]
-                fig.add_trace(
-                    go.Scatter(
-                        name=f"{category} ({column_name})",
-                        x=list(range(len(column_data))),
-                        y=scores,
-                        mode="lines+markers",
-                        marker=dict(size=5),
-                        hoverinfo="y+name",
-                        line=dict(color=category_colors[category], width=1.5),
-                        showlegend=False,
-                    ),
-                    row=row,
-                    col=col,
-                )
-            row_offset += 2
-
-        subplot_height = 350
-        total_height = total_rows * subplot_height + 200
-
-        fig.update_layout(title_text="Regard Scores", height=total_height)
-        fig.update_yaxes(range=[0, 1])
-        fig.update_xaxes(showticklabels=False, row=1, col=1)
-        fig.update_xaxes(title_text="Index", showticklabels=True, row=1, col=1)
-        fig.update_yaxes(title_text="Score", showticklabels=True, row=1, col=1)
-
-        return fig
-
-    def run(self):
-        fig = self.regard_line_plot()
-        return self.cache_results(
-            figures=[Figure(for_object=self, key=self.key, figure=fig)]
-        )
+            figures.append(hist_fig)
+
+            # Bar Chart
+            bar_fig = go.Figure(data=[go.Bar(x=df.index, y=df[category])])
+            bar_fig.update_layout(
+                title=f"{title} - {category.capitalize()} Bar Chart",
+                xaxis_title="Text Instance Index",
+                yaxis_title=category.capitalize(),
+            )
+            figures.append(bar_fig)
+
+    # Create figures for each regard score dataframe
+    create_figures(true_df, "True Text Regard")
+    create_figures(pred_df, "Predicted Text Regard")
+
+    # Calculate statistics for each regard score dataframe
+    def calculate_stats(df, metric_name):
+        stats = df.describe().loc[["mean", "50%", "max", "min", "std"]].T
+        stats.columns = [
+            "Mean Score",
+            "Median Score",
+            "Max Score",
+            "Min Score",
+            "Standard Deviation",
+        ]
+        stats["Metric"] = metric_name
+        stats["Count"] = len(df)
+        return stats
+
+    true_stats = calculate_stats(true_df, "True Text Regard")
+    pred_stats = calculate_stats(pred_df, "Predicted Text Regard")
+
+    # Combine statistics into a single dataframe
+    result_df = (
+        pd.concat([true_stats, pred_stats])
+        .reset_index()
+        .rename(columns={"index": "Category"})
+    )
+    result_df = result_df[
+        [
+            "Metric",
+            "Category",
+            "Mean Score",
+            "Median Score",
+            "Max Score",
+            "Min Score",
+            "Standard Deviation",
+            "Count",
+        ]
+    ]
+
+    return (result_df, *tuple(figures))

validmind/tests/model_validation/RougeScore.py

@@ -0,0 +1,118 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+import plotly.graph_objects as go
+from rouge import Rouge
+
+from validmind import tags, tasks
+
+
+@tags("nlp", "text_data", "visualization")
+@tasks("text_classification", "text_summarization")
+def RougeScore(dataset, model, metric="rouge-1"):
+    """
+    Evaluates the quality of machine-generated text using ROUGE metrics and visualizes the results through histograms
+    and bar charts, alongside compiling a comprehensive table of descriptive statistics for each ROUGE metric.
+
+    **Purpose:**
+    This function is designed to assess the quality of text generated by machine learning models using various ROUGE metrics.
+    ROUGE, which stands for Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics used to evaluate the
+    overlap of n-grams, word sequences, and word pairs between the machine-generated text and reference texts. This evaluation
+    is crucial for tasks such as text summarization, machine translation, and text generation, where the goal is to produce text
+    that accurately reflects the content and meaning of human-crafted references.
+
+    **Test Mechanism:**
+    The function starts by extracting the true and predicted values from the provided dataset and model. It then initializes the ROUGE
+    evaluator with the specified metric (e.g., ROUGE-1). For each pair of true and predicted texts, the function calculates the ROUGE
+    scores and compiles them into a dataframe. Histograms and bar charts are generated for each ROUGE metric (Precision, Recall, and F1 Score)
+    to visualize their distribution. Additionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum)
+    is compiled for each metric, providing a comprehensive summary of the model's performance.
+
+    **Signs of High Risk:**
+
+    - Consistently low scores across ROUGE metrics could indicate poor quality in the generated text, suggesting that the model fails
+      to capture the essential content of the reference texts.
+    - Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
+    - Low recall scores may indicate that important information from the reference text is being omitted.
+    - An imbalanced performance between precision and recall, reflected by a low F1 Score, could signal issues in the model's ability
+      to balance informativeness and conciseness.
+
+    **Strengths:**
+
+    - Provides a multifaceted evaluation of text quality through different ROUGE metrics, offering a detailed view of model performance.
+    - Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the scores.
+    - Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
+
+    **Limitations:**
+
+    - ROUGE metrics primarily focus on n-gram overlap and may not fully capture semantic coherence, fluency, or grammatical quality of the text.
+    - The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
+    - While useful for comparison, ROUGE scores alone do not provide a complete assessment of a model's performance and should be
+      supplemented with other metrics and qualitative analysis.
+    """
+
+    # Extract true and predicted values
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Initialize Rouge with the specified metric
+    rouge = Rouge(metrics=[metric])
+
+    # Calculate ROUGE scores
+    score_list = []
+    for y_t, y_p in zip(y_true, y_pred):
+        scores = rouge.get_scores(y_p, y_t, avg=True)
+        score_list.append(scores)
+
+    # Convert scores to a dataframe
+    metrics_df = pd.DataFrame(score_list)
+    df_scores = pd.DataFrame(metrics_df[metric].tolist())
+
+    # Generate histograms and bar charts for each score type
+    score_types = ["p", "r", "f"]
+    score_names = ["Precision", "Recall", "F1 Score"]
+    figures = []
+
+    for score_type, score_name in zip(score_types, score_names):
+        # Histogram
+        hist_fig = go.Figure(data=[go.Histogram(x=df_scores[score_type])])
+        hist_fig.update_layout(
+            title=f"{score_name} Histogram for {metric.upper()}",
+            xaxis_title=score_name,
+            yaxis_title="Count",
+        )
+        figures.append(hist_fig)
+
+        # Bar Chart
+        bar_fig = go.Figure(data=[go.Bar(x=df_scores.index, y=df_scores[score_type])])
+        bar_fig.update_layout(
+            title=f"{score_name} Bar Chart for {metric.upper()}",
+            xaxis_title="Row Index",
+            yaxis_title=score_name,
+        )
+        figures.append(bar_fig)
+
+    # Calculate statistics for each score type
+    stats_df = df_scores.describe().loc[["mean", "50%", "max", "min", "std"]]
+    stats_df = stats_df.rename(
+        index={
+            "mean": "Mean Score",
+            "50%": "Median Score",
+            "max": "Max Score",
+            "min": "Min Score",
+            "std": "Standard Deviation",
+        }
+    ).T
+    stats_df["Count"] = len(df_scores)
+
+    # Rename metrics for clarity
+    stats_df.index = stats_df.index.map(
+        {"p": "Precision", "r": "Recall", "f": "F1 Score"}
+    )
+
+    # Create a DataFrame from all collected statistics
+    result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
+
+    return (result_df, *tuple(figures))