validmind 2.1.0__py3-none-any.whl → 2.2.2__py3-none-any.whl

validmind/tests/model_validation/RougeScore.py

@@ -0,0 +1,118 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+import plotly.graph_objects as go
+from rouge import Rouge
+
+from validmind import tags, tasks
+
+
+@tags("nlp", "text_data", "visualization")
+@tasks("text_classification", "text_summarization")
+def RougeScore(dataset, model, metric="rouge-1"):
+    """
+    Evaluates the quality of machine-generated text using ROUGE metrics and visualizes the results through histograms
+    and bar charts, alongside compiling a comprehensive table of descriptive statistics for each ROUGE metric.
+
+    **Purpose:**
+    This function is designed to assess the quality of text generated by machine learning models using various ROUGE metrics.
+    ROUGE, which stands for Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics used to evaluate the
+    overlap of n-grams, word sequences, and word pairs between the machine-generated text and reference texts. This evaluation
+    is crucial for tasks such as text summarization, machine translation, and text generation, where the goal is to produce text
+    that accurately reflects the content and meaning of human-crafted references.
+
+    **Test Mechanism:**
+    The function starts by extracting the true and predicted values from the provided dataset and model. It then initializes the ROUGE
+    evaluator with the specified metric (e.g., ROUGE-1). For each pair of true and predicted texts, the function calculates the ROUGE
+    scores and compiles them into a dataframe. Histograms and bar charts are generated for each ROUGE metric (Precision, Recall, and F1 Score)
+    to visualize their distribution. Additionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum)
+    is compiled for each metric, providing a comprehensive summary of the model's performance.
+
+    **Signs of High Risk:**
+
+    - Consistently low scores across ROUGE metrics could indicate poor quality in the generated text, suggesting that the model fails
+      to capture the essential content of the reference texts.
+    - Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
+    - Low recall scores may indicate that important information from the reference text is being omitted.
+    - An imbalanced performance between precision and recall, reflected by a low F1 Score, could signal issues in the model's ability
+      to balance informativeness and conciseness.
+
+    **Strengths:**
+
+    - Provides a multifaceted evaluation of text quality through different ROUGE metrics, offering a detailed view of model performance.
+    - Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the scores.
+    - Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
+
+    **Limitations:**
+
+    - ROUGE metrics primarily focus on n-gram overlap and may not fully capture semantic coherence, fluency, or grammatical quality of the text.
+    - The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
+    - While useful for comparison, ROUGE scores alone do not provide a complete assessment of a model's performance and should be
+      supplemented with other metrics and qualitative analysis.
+    """
+
+    # Extract true and predicted values
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Initialize Rouge with the specified metric
+    rouge = Rouge(metrics=[metric])
+
+    # Calculate ROUGE scores
+    score_list = []
+    for y_t, y_p in zip(y_true, y_pred):
+        scores = rouge.get_scores(y_p, y_t, avg=True)
+        score_list.append(scores)
+
+    # Convert scores to a dataframe
+    metrics_df = pd.DataFrame(score_list)
+    df_scores = pd.DataFrame(metrics_df[metric].tolist())
+
+    # Generate histograms and bar charts for each score type
+    score_types = ["p", "r", "f"]
+    score_names = ["Precision", "Recall", "F1 Score"]
+    figures = []
+
+    for score_type, score_name in zip(score_types, score_names):
+        # Histogram
+        hist_fig = go.Figure(data=[go.Histogram(x=df_scores[score_type])])
+        hist_fig.update_layout(
+            title=f"{score_name} Histogram for {metric.upper()}",
+            xaxis_title=score_name,
+            yaxis_title="Count",
+        )
+        figures.append(hist_fig)
+
+        # Bar Chart
+        bar_fig = go.Figure(data=[go.Bar(x=df_scores.index, y=df_scores[score_type])])
+        bar_fig.update_layout(
+            title=f"{score_name} Bar Chart for {metric.upper()}",
+            xaxis_title="Row Index",
+            yaxis_title=score_name,
+        )
+        figures.append(bar_fig)
+
+    # Calculate statistics for each score type
+    stats_df = df_scores.describe().loc[["mean", "50%", "max", "min", "std"]]
+    stats_df = stats_df.rename(
+        index={
+            "mean": "Mean Score",
+            "50%": "Median Score",
+            "max": "Max Score",
+            "min": "Min Score",
+            "std": "Standard Deviation",
+        }
+    ).T
+    stats_df["Count"] = len(df_scores)
+
+    # Rename metrics for clarity
+    stats_df.index = stats_df.index.map(
+        {"p": "Precision", "r": "Recall", "f": "F1 Score"}
+    )
+
+    # Create a DataFrame from all collected statistics
+    result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
+
+    return (result_df, *tuple(figures))

validmind/tests/model_validation/TokenDisparity.py

@@ -2,139 +2,102 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-import itertools
-from dataclasses import dataclass
-
 import pandas as pd
 import plotly.graph_objects as go
-from plotly.subplots import make_subplots
-from transformers import BertTokenizer
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
 
 
-@dataclass
-class TokenDisparity(Metric):
+@tags("nlp", "text_data", "visualization")
+@tasks("text_classification", "text_summarization")
+def TokenDisparity(dataset, model):
     """
-    Assess and visualize token count disparity between model's predicted and actual dataset.
-
-    **Purpose**:
-    The Token Disparity metric is designed to assess the distributional congruence between the model's predicted
-    outputs and the actual data. This is achieved by constructing histograms that illustrate the disparity in token
-    count between the two columns. Additionally, this metric is used to measure the model's verbosity in comparison to
-    the genuine dataset.
-
-    **Test Mechanism**:
-    The mechanism of running this test involves tokenizing both columns: one containing the actual data and the other
-    containing the model's predictions. The BERT tokenizer is used for tokenizing the contents of each column. After
-    tokenization, tokens in each column are counted and represented in two distinct histograms to facilitate the
-    visualization of token count distribution in the actual and predicted data. To quantify the difference in
-    distribution, the histogram of the actual tokens is compared with the histogram of the predicted tokens.
-
-    **Signs of High Risk**:
-    High risk or potential failure in model performance may be suggested by:
-
-    - Significant incongruities in distribution patterns between the two histograms.
-    - Marked divergence of the predicted histogram from the reference histogram, indicating that the model may be
-    generating output with unexpected verbosity.
-    - This might result in an output that has a significantly higher or lower number of tokens than expected.
-
-    **Strengths**:
-    Strengths of the Token Disparity metric include:
-
-    - It provides a clear and visual comparison of predicted versus actual token distributions, enhancing understanding
-    of the model's output consistency and verbosity.
-    - It is able to detect potential issues with the model's output generation capability, such as over-production or
-    under-production of tokens compared to the actual data set.
-
-    **Limitations**:
-    Limitations of the Token Disparity metric include:
-
-    - The metric focuses solely on token count, disregarding the semantics behind those tokens. Consequently, it may
-    miss out on issues related to relevance or meaningfulness of produced tokens.
-    - The assumption that similar token count between predicted and actual data suggests accurate output, which is not
-    always the case.
-    - Dependence on the BERT tokenizer, which may not always be the optimum choice for all types of text data.
+    Evaluates the token disparity between reference and generated texts, visualizing the results through histograms
+    and bar charts, alongside compiling a comprehensive table of descriptive statistics for token counts.
+
+    **Purpose:**
+    This function is designed to assess the token disparity between reference and generated texts. Token disparity is
+    important for understanding how closely the length and token usage of generated texts match the reference texts.
+
+    **Test Mechanism:**
+    The function starts by extracting the true and predicted values from the provided dataset and model. It then calculates
+    the number of tokens in each reference and generated text. Histograms and bar charts are generated for the token counts
+    of both reference and generated texts to visualize their distribution. Additionally, a table of descriptive statistics
+    (mean, median, standard deviation, minimum, and maximum) is compiled for the token counts, providing a comprehensive
+    summary of the model's performance.
+
+    **Signs of High Risk:**
+    - Significant disparity in token counts between reference and generated texts could indicate issues with text generation
+      quality, such as verbosity or lack of detail.
+    - Consistently low token counts in generated texts compared to references might suggest that the model is producing
+      incomplete or overly concise outputs.
+
+    **Strengths:**
+    - Provides a simple yet effective evaluation of text length and token usage.
+    - Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of token counts.
+    - Descriptive statistics offer a concise summary of the model's performance in generating texts of appropriate length.
+
+    **Limitations:**
+    - Token counts alone do not provide a complete assessment of text quality and should be supplemented with other metrics and qualitative analysis.
     """
 
-    name = "token_disparity"
-    required_inputs = ["model", "dataset"]
-
-    def run(self):
-        y_true = list(itertools.chain.from_iterable(self.inputs.dataset.y))
-        y_pred = self.inputs.dataset.y_pred(self.inputs.model)
-
-        df = pd.DataFrame({"reference_column": y_true, "generated_column": y_pred})
-
-        fig = self.token_disparity_histograms(df)
-        figures = []
-        figures.append(
-            Figure(
-                for_object=self,
-                key=self.key,
-                figure=fig,
-            )
-        )
-        return self.cache_results(figures=figures)
-
-    def token_disparity_histograms(self, df):
-        """
-        Visualize the token counts distribution of two given columns using histograms.
-
-        :param df: DataFrame containing the text columns.
-        :param params: Dictionary with the keys ["reference_column", "generated_column"].
-        """
+    # Extract true and predicted values
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
 
-        reference_column = "reference_column"
-        generated_column = "generated_column"
+    # Calculate token counts
+    token_counts_true = [len(text.split()) for text in y_true]
+    token_counts_pred = [len(text.split()) for text in y_pred]
 
-        # Initialize the tokenizer
-        tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
+    # Create a dataframe for reference and generated token counts
+    df = pd.DataFrame(
+        {"reference_tokens": token_counts_true, "generated_tokens": token_counts_pred}
+    )
 
-        # Tokenize the columns and get the number of tokens
-        df["tokens_1"] = df[reference_column].apply(
-            lambda x: len(tokenizer.tokenize(x))
-        )
-        df["tokens_2"] = df[generated_column].apply(
-            lambda x: len(tokenizer.tokenize(x))
-        )
+    figures = []
 
-        # Create subplots: 1 row, 2 columns
-        fig = make_subplots(
-            rows=1,
-            cols=2,
-            subplot_titles=(
-                f"Tokens in {reference_column}",
-                f"Tokens in {generated_column}",
-            ),
-        )
+    # Generate histograms and bar charts for reference and generated token counts
+    token_types = ["reference_tokens", "generated_tokens"]
+    token_names = ["Reference Tokens", "Generated Tokens"]
 
-        # Add histograms
-        fig.add_trace(
-            go.Histogram(
-                x=df["tokens_1"],
-                marker_color="blue",
-                name=f"Tokens in {reference_column}",
-            ),
-            row=1,
-            col=1,
+    for token_type, token_name in zip(token_types, token_names):
+        # Histogram
+        hist_fig = go.Figure(data=[go.Histogram(x=df[token_type])])
+        hist_fig.update_layout(
+            title=f"{token_name} Histogram",
+            xaxis_title=token_name,
+            yaxis_title="Count",
         )
-
-        fig.add_trace(
-            go.Histogram(
-                x=df["tokens_2"],
-                marker_color="red",
-                name=f"Tokens in {generated_column}",
-            ),
-            row=1,
-            col=2,
+        figures.append(hist_fig)
+
+        # Bar Chart
+        bar_fig = go.Figure(data=[go.Bar(x=df.index, y=df[token_type])])
+        bar_fig.update_layout(
+            title=f"{token_name} Bar Chart",
+            xaxis_title="Row Index",
+            yaxis_title=token_name,
         )
-
-        # Update layout
-        fig.update_layout(title_text="Token Distributions", bargap=0.1)
-
-        fig.update_yaxes(title_text="Number of Documents")
-        fig.update_xaxes(title_text="Number of Tokens", row=1, col=1)
-        fig.update_xaxes(title_text="Number of Tokens", row=1, col=2)
-
-        return fig
+        figures.append(bar_fig)
+
+    # Calculate statistics for each token count type
+    stats_df = df.describe().loc[["mean", "50%", "max", "min", "std"]]
+    stats_df = stats_df.rename(
+        index={
+            "mean": "Mean Count",
+            "50%": "Median Count",
+            "max": "Max Count",
+            "min": "Min Count",
+            "std": "Standard Deviation",
+        }
+    ).T
+    stats_df["Count"] = len(df)
+
+    # Rename columns for clarity
+    stats_df.index = stats_df.index.map(
+        {"reference_tokens": "Reference Tokens", "generated_tokens": "Generated Tokens"}
+    )
+
+    # Create a DataFrame from all collected statistics
+    result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
+
+    return (result_df, *tuple(figures))

validmind/tests/model_validation/ToxicityScore.py

@@ -2,146 +2,132 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-import itertools
-from dataclasses import dataclass
-
 import evaluate
 import pandas as pd
 import plotly.graph_objects as go
-import plotly.subplots as sp
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
 
 
-@dataclass
-class ToxicityScore(Metric):
+@tags("nlp", "text_data", "visualization")
+@tasks("text_classification", "text_summarization")
+def ToxicityScore(dataset, model):
     """
+    Computes and visualizes the toxicity score for input text, true text, and predicted text, assessing content quality and potential risk.
+
     **Purpose:**
-    The ToxicityScore metric is designed to present a sequential representation of toxicity scores for various texts.
-    Leveraging line plots, it gives an overview of how toxicity scores evolve across the sequence of texts, highlighting
-    trends and patterns.
+    The ToxicityScore metric is designed to evaluate the toxicity levels of texts generated by models. This is crucial for
+    identifying and mitigating harmful or offensive content in machine-generated texts.
 
     **Test Mechanism:**
-    The mechanism involves fetching texts from specific columns, computing their toxicity scores using a preloaded
-    `toxicity` evaluation tool, and then plotting these scores. A multi-panel visualization is created where each
-    panel is dedicated to a specific text data column. Line plots serve as the primary visual tool, showing the progression of toxicity scores across text sequences. Each
-    line plot corresponds to a specific text data column, illustrating the variation in toxicity scores as one moves
-    from one text segment to the next.
+    The function starts by extracting the input, true, and predicted values from the provided dataset and model. The toxicity score is
+    computed for each text using a preloaded `toxicity` evaluation tool. The scores are compiled into dataframes, and histograms
+    and bar charts are generated to visualize the distribution of toxicity scores. Additionally, a table of descriptive statistics
+    (mean, median, standard deviation, minimum, and maximum) is compiled for the toxicity scores, providing a comprehensive
+    summary of the model's performance.
 
     **Signs of High Risk:**
-    Drastic spikes in the line plots, especially those that reach high toxicity values, indicate potentially toxic
-    content within the associated text segment. If predicted summaries diverge significantly from input or target
-    texts, it could be indicative of issues in the model's generated content.
+    - Drastic spikes in toxicity scores indicate potentially toxic content within the associated text segment.
+    - Persistent high toxicity scores across multiple texts may suggest systemic issues in the model's text generation process.
 
     **Strengths:**
-    The ToxicityScore offers a dynamic view of toxicity trends, enabling users to detect patterns or irregularities
-    across the dataset. This is particularly valuable when comparing predicted content with actual data, helping
-    highlight any inconsistencies or abnormalities in model output.
+    - Provides a clear evaluation of toxicity levels in generated texts, helping to ensure content safety and appropriateness.
+    - Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of toxicity scores.
+    - Descriptive statistics offer a concise summary of the model's performance in generating non-toxic texts.
 
     **Limitations:**
-    This metric’s accuracy is contingent upon the underlying `toxicity` tool. The line plots provide a broad overview
-    of toxicity trends but do not specify which portions or tokens of the text are responsible for high toxicity scores.
-    Consequently, for granular insights, supplementary, in-depth analysis might be needed.
+    - The accuracy of the toxicity scores is contingent upon the underlying `toxicity` tool.
+    - The scores provide a broad overview but do not specify which portions or tokens of the text are responsible for high toxicity.
+    - Supplementary, in-depth analysis might be needed for granular insights.
     """
 
-    name = "toxicity_line_plot"
-    required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": [
-            "text_classification",
-            "text_summarization",
-        ],
-        "tags": ["toxicity_line_plot"],
-    }
-
-    def _get_datasets(self):
-        # Check model attributes
-        if not hasattr(self, "model"):
-            raise AttributeError("The 'model' attribute is missing.")
-
-        y_true = list(itertools.chain.from_iterable(self.inputs.dataset.y))
-        y_pred = self.inputs.dataset.y_pred(self.inputs.model)
-        input_text = self.inputs.dataset.df[self.inputs.dataset.text_column]
-
-        # Ensure consistency in lengths
-        if not len(y_true) == len(y_pred) == len(input_text):
-            raise ValueError(
-                "Inconsistent lengths among input text, true summaries, and predicted summaries."
-            )
-
-        return input_text, y_true, y_pred
-
-    def toxicity_line_plots(self, df):
-        """
-        Compute toxicity scores for texts and then plot line plots for all columns of df.
-
-        Parameters:
-        - df (pd.DataFrame): The dataframe containing texts.
-        """
-
-        # Extract necessary parameters
-        toxicity = evaluate.load("toxicity")
-
-        # Get all columns of df
-        text_columns = df.columns.tolist()
-
-        # Determine the number of rows required based on the number of text columns
-        num_rows = (len(text_columns) + 1) // 2
-
-        # Create a subplot layout
-        fig = sp.make_subplots(rows=num_rows, cols=2, subplot_titles=text_columns)
-
-        subplot_height = 350
-        total_height = num_rows * subplot_height + 200
-
-        for idx, col in enumerate(text_columns, start=1):
-            row = (idx - 1) // 2 + 1
-            col_idx = (idx - 1) % 2 + 1
-
-            # Get list of texts from dataframe
-            texts = df[col].tolist()
-
-            # Compute toxicity for texts
-            toxicity_scores = toxicity.compute(predictions=texts)["toxicity"]
-
-            # Add traces to the corresponding subplot
-            fig.add_trace(
-                go.Scatter(
-                    y=toxicity_scores,
-                    mode="lines+markers",
-                    marker=dict(size=5),
-                    line=dict(width=1.5),
-                    showlegend=False,
-                ),
-                row=row,
-                col=col_idx,
-            )
-
-            # Update xaxes and yaxes titles only for the first subplot
-            if idx == 1:
-                fig.update_xaxes(title_text="Text Index", row=row, col=col_idx)
-                fig.update_yaxes(title_text="Toxicity Score", row=row, col=col_idx)
-
-        # Update layout
-        fig.update_layout(
-            title_text="Line Plots of Toxicity Scores", height=total_height
-        )
-
-        return fig
-
-    def run(self):
-        input_text, y_true, y_pred = self._get_datasets()
-
-        df = pd.DataFrame(
-            {
-                "Input Text": input_text,
-                "Target Text": y_true,
-                "Predicted Summaries": y_pred,
-            }
+    # Extract true, predicted, and input values
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+    input_text = dataset.df[dataset.text_column]
+
+    # Load the toxicity evaluation metric
+    toxicity = evaluate.load("toxicity")
+
+    # Function to calculate toxicity scores
+    def compute_toxicity_scores(texts):
+        scores = []
+        for text in texts:
+            score = toxicity.compute(predictions=[text])
+            scores.append(score["toxicity"])
+        return scores
+
+    # Calculate toxicity scores for input, true, and predicted texts
+    input_toxicity = compute_toxicity_scores(input_text)
+    true_toxicity = compute_toxicity_scores(y_true)
+    pred_toxicity = compute_toxicity_scores(y_pred)
+
+    # Convert scores to dataframes
+    input_df = pd.DataFrame(input_toxicity, columns=["Input Text Toxicity"])
+    true_df = pd.DataFrame(true_toxicity, columns=["True Text Toxicity"])
+    pred_df = pd.DataFrame(pred_toxicity, columns=["Predicted Text Toxicity"])
+
+    figures = []
+
+    # Function to create histogram and bar chart for toxicity scores
+    def create_figures(df, title):
+        # Histogram
+        hist_fig = go.Figure(data=[go.Histogram(x=df.iloc[:, 0])])
+        hist_fig.update_layout(
+            title=f"{title} Histogram",
+            xaxis_title=title,
+            yaxis_title="Count",
         )
-
-        fig = self.toxicity_line_plots(df)
-
-        return self.cache_results(
-            figures=[Figure(for_object=self, key=self.key, figure=fig)]
+        figures.append(hist_fig)
+
+        # Bar Chart
+        bar_fig = go.Figure(data=[go.Bar(x=df.index, y=df.iloc[:, 0])])
+        bar_fig.update_layout(
+            title=f"{title} Bar Chart",
+            xaxis_title="Text Instance Index",
+            yaxis_title=title,
         )
+        figures.append(bar_fig)
+
+    # Create figures for each toxicity score dataframe
+    create_figures(input_df, "Input Text Toxicity")
+    create_figures(true_df, "True Text Toxicity")
+    create_figures(pred_df, "Predicted Text Toxicity")
+
+    # Calculate statistics for each toxicity score dataframe
+    def calculate_stats(df):
+        stats = df.describe().loc[["mean", "50%", "max", "min", "std"]].T
+        stats.columns = [
+            "Mean Score",
+            "Median Score",
+            "Max Score",
+            "Min Score",
+            "Standard Deviation",
+        ]
+        stats["Metric"] = df.columns[0]
+        stats["Count"] = len(df)
+        return stats
+
+    input_stats = calculate_stats(input_df)
+    true_stats = calculate_stats(true_df)
+    pred_stats = calculate_stats(pred_df)
+
+    # Combine statistics into a single dataframe
+    result_df = (
+        pd.concat([input_stats, true_stats, pred_stats])
+        .reset_index()
+        .rename(columns={"index": "Statistic"})
+    )
+    result_df = result_df[
+        [
+            "Metric",
+            "Mean Score",
+            "Median Score",
+            "Max Score",
+            "Min Score",
+            "Standard Deviation",
+            "Count",
+        ]
+    ]
+
+    return (result_df, *tuple(figures))