validmind 2.5.25__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/tests/data_validation/nlp/StopWords.py

@@ -7,26 +7,21 @@ Threshold based tests
 """
 
 from collections import defaultdict
-from dataclasses import dataclass
-from typing import List
 
-import matplotlib.pyplot as plt
 import nltk
 import pandas as pd
+import plotly.graph_objects as go
 from nltk.corpus import stopwords
 
-from validmind.vm_models import (
-    Figure,
-    ResultSummary,
-    ResultTable,
-    ResultTableMetadata,
-    ThresholdTest,
-    ThresholdTestResult,
-)
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset
 
 
-@dataclass
-class StopWords(ThresholdTest):
+@tags("nlp", "text_data", "frequency_analysis", "visualization")
+@tasks("text_classification", "text_summarization")
+def StopWords(
+    dataset: VMDataset, min_percent_threshold: float = 0.5, num_words: int = 25
+):
     """
     Evaluates and visualizes the frequency of English stop words in a text dataset against a defined threshold.
 
@@ -75,82 +70,58 @@ class StopWords(ThresholdTest):
     or predictive accuracy.
     """
 
-    name = "stop_words"
-    required_inputs = ["dataset"]
-    default_params = {"min_percent_threshold": 0.5, "num_words": 25}
-    tasks = ["text_classification", "text_summarization"]
-    tags = ["nlp", "text_data", "visualization", "frequency_analysis"]
-
-    def summary(self, results: List[ThresholdTestResult], all_passed: bool):
-        # Create a DataFrame from the data
-        df = pd.DataFrame(results[0].values, columns=["Word", "Percentage"])
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=df,
-                    metadata=ResultTableMetadata(
-                        title=f"Stop words results for column '{self.inputs.dataset.target_column}'"
-                    ),
-                )
-            ]
-        )
-
-    def run(self):
-        text_column = self.inputs.dataset.text_column
-
-        def create_corpus(df, text_column):
-            corpus = []
-            for x in df[text_column].str.split():
-                for i in x:
-                    corpus.append(i)
-            return corpus
-
-        corpus = create_corpus(self.inputs.dataset.df, text_column=text_column)
-
-        nltk.download("stopwords")
-        stop = set(stopwords.words("english"))
-        dic = defaultdict(int)
-        for word in corpus:
-            if word in stop:
-                dic[word] += 1
-        # Calculate the total number of words in the corpus
-        total_words = len(corpus)
-
-        # Calculate the percentage of each word in the corpus
-        word_percentages = {}
-        for word, count in dic.items():
-            percentage = (count / total_words) * 100
-            word_percentages[word] = percentage
-
-        passed = all(word_percentages.values()) < self.params["min_percent_threshold"]
-        top = sorted(word_percentages.items(), key=lambda x: x[1], reverse=True)[
-            : self.params["num_words"]
-        ]
-
-        test_results = [
-            ThresholdTestResult(
-                passed=passed,
-                values=top,
-            )
-        ]
-        figures = []
-        if top:
-            fig, _ = plt.subplots()
-            x, y = zip(*top)
-            plt.bar(x, y)
-            plt.xticks(rotation=90)
-
-            # Do this if you want to prevent the figure from being displayed
-            plt.close("all")
-
-            figures = []
-            figures.append(
-                Figure(
-                    for_object=self,
-                    key=f"{self.name}",
-                    figure=fig,
-                )
-            )
+    text_column = dataset.text_column
+
+    def create_corpus(df, text_column):
+        corpus = []
+        for x in df[text_column].str.split():
+            for i in x:
+                corpus.append(i)
+        return corpus
+
+    corpus = create_corpus(dataset.df, text_column=text_column)
+
+    nltk.download("stopwords", quiet=True)
+
+    stop = set(stopwords.words("english"))
+    dic = defaultdict(int)
+    for word in corpus:
+        if word in stop:
+            dic[word] += 1
+
+    # Calculate the total number of words in the corpus
+    total_words = len(corpus)
 
-        return self.cache_results(test_results, passed=passed, figures=figures)
+    # Calculate the percentage of each word in the corpus
+    word_percentages = {}
+    for word, count in dic.items():
+        percentage = (count / total_words) * 100
+        word_percentages[word] = percentage
+
+    passed = all(word_percentages.values()) < min_percent_threshold
+    results = sorted(word_percentages.items(), key=lambda x: x[1], reverse=True)[
+        :num_words
+    ]
+
+    if not results:
+        return passed
+
+    x, y = zip(*results)
+
+    fig = go.Figure(data=[go.Bar(x=x, y=y)])
+    fig.update_layout(
+        title=f"Stop Words Frequency in '{text_column}'",
+        xaxis_title="Stop Words",
+        yaxis_title="Percentage (%)",
+        xaxis_tickangle=-45,
+    )
+
+    return (
+        {
+            f"Stop words results for column '{text_column}'": pd.DataFrame(
+                results, columns=["Word", "Percentage"]
+            )
+        },
+        fig,
+        passed,
+    )

validmind/tests/data_validation/nlp/TextDescription.py

@@ -3,19 +3,98 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import string
-from dataclasses import dataclass
 
-import matplotlib.pyplot as plt
 import nltk
 import pandas as pd
 import plotly.express as px
 from nltk.corpus import stopwords
 
-from ....vm_models import Figure, Metric, VMDataset
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset
 
 
-@dataclass
-class TextDescription(Metric):
+def create_metrics_df(df, text_column, unwanted_tokens, lang):
+    stop_words = set(word.lower() for word in stopwords.words(lang))
+    unwanted_tokens = set(token.lower() for token in unwanted_tokens)
+
+    results = []
+
+    for text in df[text_column]:
+        # pre-process text
+        words = nltk.word_tokenize(text)
+        filtered_words = [
+            word
+            for word in words
+            if word.lower() not in stop_words
+            and word.lower() not in unwanted_tokens
+            and word not in string.punctuation
+        ]
+        sentences = nltk.sent_tokenize(text)
+
+        # calculate metrics
+        total_words = len(filtered_words)
+        total_sentences = len(sentences)
+        avg_sentence_length = round(
+            (
+                sum(len(sentence.split()) for sentence in sentences) / total_sentences
+                if total_sentences
+                else 0
+            ),
+            1,
+        )
+        total_paragraphs = len(text.split("\n\n"))
+        total_unique_words = len(set(filtered_words))
+        total_punctuations = sum(1 for word in words if word in string.punctuation)
+        lexical_diversity = round(
+            total_unique_words / len(filtered_words) if filtered_words else 0, 1
+        )
+
+        results.append(
+            [
+                total_words,
+                total_sentences,
+                avg_sentence_length,
+                total_paragraphs,
+                total_unique_words,
+                total_punctuations,
+                lexical_diversity,
+            ]
+        )
+
+    return pd.DataFrame(
+        results,
+        columns=[
+            "Total Words",
+            "Total Sentences",
+            "Avg Sentence Length",
+            "Total Paragraphs",
+            "Total Unique Words",
+            "Total Punctuations",
+            "Lexical Diversity",
+        ],
+    )
+
+
+@tags("nlp", "text_data", "visualization")
+@tasks("text_classification", "text_summarization")
+def TextDescription(
+    dataset: VMDataset,
+    unwanted_tokens: set = {
+        "s",
+        "s'",
+        "mr",
+        "ms",
+        "mrs",
+        "dr",
+        "'s",
+        " ",
+        "''",
+        "dollar",
+        "us",
+        "``",
+    },
+    lang: str = "english",
+):
     """
     Conducts comprehensive textual analysis on a dataset using NLTK to evaluate various parameters and generate
     visualizations.
@@ -60,160 +139,38 @@ class TextDescription(Metric):
     - Assumes well-structured documents, which may result in inaccuracies with poorly formatted text.
     """
 
-    name = "text_description"
-    required_inputs = ["dataset"]
-    default_params = {
-        "unwanted_tokens": {
-            "s",
-            "s'",
-            "mr",
-            "ms",
-            "mrs",
-            "dr",
-            "'s",
-            " ",
-            "''",
-            "dollar",
-            "us",
-            "``",
-        },
-        "num_top_words": 3,
-        "lang": "english",
-    }
-    tasks = ["text_classification", "text_summarization"]
-    tags = ["nlp", "text_data", "visualization"]
-
-    def general_text_metrics(self, df, text_column):
-        results = []
-
-        for text in df[text_column]:
-            sentences = nltk.sent_tokenize(text)
-            words = nltk.word_tokenize(text)
-            paragraphs = text.split("\n\n")
-
-            total_words = len(words)
-            total_sentences = len(sentences)
-            avg_sentence_length = round(
-                (
-                    sum(len(sentence.split()) for sentence in sentences)
-                    / total_sentences
-                    if total_sentences
-                    else 0
-                ),
-                1,
+    if dataset.text_column is None:
+        raise ValueError("A 'text_column' must be provided to run this test.")
+
+    nltk.download("punkt_tab", quiet=True)
+
+    metrics_df = create_metrics_df(
+        dataset.df, dataset.text_column, unwanted_tokens, lang
+    )
+
+    combinations_to_plot = [
+        ("Total Words", "Total Sentences"),
+        ("Total Words", "Total Unique Words"),
+        ("Total Sentences", "Avg Sentence Length"),
+        ("Total Unique Words", "Lexical Diversity"),
+    ]
+
+    figures = []
+
+    # Create hist plots for each column
+    for column in metrics_df.columns:
+        fig = px.histogram(metrics_df, x=column)
+        fig.update_layout(bargap=0.2)
+        figures.append(fig)
+
+    for metric1, metric2 in combinations_to_plot:
+        figures.append(
+            px.scatter(
+                metrics_df,
+                x=metric1,
+                y=metric2,
+                title=f"Scatter Plot: {metric1} vs {metric2}",
             )
-            total_paragraphs = len(paragraphs)
-
-            results.append(
-                [total_words, total_sentences, avg_sentence_length, total_paragraphs]
-            )
-
-        return pd.DataFrame(
-            results,
-            columns=[
-                "Total Words",
-                "Total Sentences",
-                "Avg Sentence Length",
-                "Total Paragraphs",
-            ],
         )
 
-    def vocabulary_structure_metrics(
-        self, df, text_column, unwanted_tokens, num_top_words, lang
-    ):
-        stop_words = set(word.lower() for word in stopwords.words(lang))
-        unwanted_tokens = set(token.lower() for token in unwanted_tokens)
-
-        results = []
-
-        for text in df[text_column]:
-            words = nltk.word_tokenize(text)
-
-            filtered_words = [
-                word
-                for word in words
-                if word.lower() not in stop_words
-                and word.lower() not in unwanted_tokens
-                and word not in string.punctuation
-            ]
-
-            total_unique_words = len(set(filtered_words))
-            total_punctuations = sum(1 for word in words if word in string.punctuation)
-            lexical_diversity = round(
-                total_unique_words / len(filtered_words) if filtered_words else 0, 1
-            )
-
-            results.append([total_unique_words, total_punctuations, lexical_diversity])
-
-        return pd.DataFrame(
-            results,
-            columns=["Total Unique Words", "Total Punctuations", "Lexical Diversity"],
-        )
-
-    # Wrapper function that combines the outputs
-    def text_description_table(self, df, params):
-        text_column = self.inputs.dataset.text_column
-        unwanted_tokens = params["unwanted_tokens"]
-        num_top_words = params["num_top_words"]
-        lang = params["lang"]
-
-        gen_metrics_df = self.general_text_metrics(df, text_column)
-        vocab_metrics_df = self.vocabulary_structure_metrics(
-            df, text_column, unwanted_tokens, num_top_words, lang
-        )
-        combined_df = pd.concat([gen_metrics_df, vocab_metrics_df], axis=1)
-
-        return combined_df
-
-    def run(self):
-        # Enforce that text_column must be provided as part of the params
-        if self.inputs.dataset.text_column is None:
-            raise ValueError("A 'text_column' must be provided to run this test.")
-
-        # Can only run this test if we have a Dataset object
-        if not isinstance(self.inputs.dataset, VMDataset):
-            raise ValueError("TextDescription requires a validmind Dataset object")
-
-        # download nltk data
-        nltk.download("punkt_tab", quiet=True)
-
-        df_text_description = self.text_description_table(
-            self.inputs.dataset.df, self.params
-        )
-
-        # Define the combinations you want to plot
-        combinations_to_plot = [
-            ("Total Words", "Total Sentences"),
-            ("Total Words", "Total Unique Words"),
-            ("Total Sentences", "Avg Sentence Length"),
-            ("Total Unique Words", "Lexical Diversity"),
-        ]
-        params = {"combinations_to_plot": combinations_to_plot}
-        figures = self.text_description_plots(df_text_description, params)
-
-        return self.cache_results(
-            figures=figures,
-        )
-
-    # Function to plot scatter plots for specified combinations using Plotly
-    def text_description_plots(self, df, params):
-        combinations_to_plot = params["combinations_to_plot"]
-        figures = []
-        # Create hist plots for each column
-        for i, column in enumerate(df.columns):
-            fig = px.histogram(df, x=column)
-            fig.update_layout(bargap=0.2)
-            # Generate a unique key for each histogram using the column name and index
-            histogram_key = f"{self.name}_histogram_{column}_{i}"
-            figures.append(Figure(for_object=self, key=histogram_key, figure=fig))
-
-        for j, (metric1, metric2) in enumerate(combinations_to_plot):
-            fig = px.scatter(
-                df, x=metric1, y=metric2, title=f"Scatter Plot: {metric1} vs {metric2}"
-            )
-            # Generate a unique key for each scatter plot using the metric names and index
-            scatter_key = f"{self.name}_scatter_{metric1}_vs_{metric2}_{j}"
-            figures.append(Figure(for_object=self, key=scatter_key, figure=fig))
-        plt.close("all")
-
-        return figures
+    return tuple(figures)

validmind/tests/data_validation/nlp/Toxicity.py

@@ -49,9 +49,15 @@ def Toxicity(dataset):
     - Does not provide context-specific insights, which may be necessary for nuanced understanding.
     - May not capture all forms of subtle or indirect toxic language.
     """
+
+    # Check text column
+    if not dataset.text_column:
+        raise ValueError("Please set text_column name in the Validmind Dataset object")
+
+    text_inputs = dataset.df[dataset.text_column].tolist()
+
     toxicity = evaluate.load("toxicity")
-    input_text = dataset.df[dataset.text_column]
-    toxicity_scores = toxicity.compute(predictions=list(input_text.values))["toxicity"]
+    toxicity_scores = toxicity.compute(predictions=text_inputs)["toxicity"]
 
     fig = plt.figure()
     ax = sns.kdeplot(
@@ -62,7 +68,9 @@ def Toxicity(dataset):
         alpha=0.5,
         linewidth=0,
     )
-    ax.set_title(f"Toxicity score of {dataset.text_column} ")
+    ax.set_title(f"Toxicity score of {dataset.text_column}")
     ax.set_xlabel("Toxicity score")
-    plt.close("all")
+
+    plt.close()
+
     return fig