validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/tests/data_validation/ClassImbalance.py

@@ -5,26 +5,20 @@
 """
 Threshold based tests
 """
-from dataclasses import dataclass
-from typing import List
+from typing import Any, Dict, Tuple
 
-import pandas as pd
 import plotly.graph_objs as go
 
+from validmind import tags, tasks
 from validmind.errors import SkipTestError
-from validmind.vm_models import (
-    Figure,
-    ResultSummary,
-    ResultTable,
-    ResultTableMetadata,
-    ThresholdTest,
-    ThresholdTestResult,
-    VMDataset,
-)
-
-
-@dataclass
-class ClassImbalance(ThresholdTest):
+from validmind.vm_models import VMDataset
+
+
+@tags("tabular_data", "binary_classification", "multiclass_classification")
+@tasks("classification")
+def ClassImbalance(
+    dataset: VMDataset, min_percent_threshold: int = 10
+) -> Tuple[Dict[str, Any], go.Figure, bool]:
     """
     Evaluates and quantifies class distribution imbalance in a dataset used by a machine learning model.
 
@@ -71,106 +65,43 @@ class ClassImbalance(ThresholdTest):
     these imbalances.
     - The test is only applicable for classification operations and unsuitable for regression or clustering tasks.
     """
-
-    # Changing the name test to avoid a name clash
-    name = "class_imbalance"
-    required_inputs = ["dataset"]
-    default_params = {"min_percent_threshold": 10}
-    tasks = ["classification"]
-    tags = ["tabular_data", "binary_classification", "multiclass_classification"]
-
-    def summary(self, results: List[ThresholdTestResult], all_passed: bool):
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=results[0].values,
-                    metadata=ResultTableMetadata(
-                        title=f"Class Imbalance Results for Column {self.inputs.dataset.target_column}"
-                    ),
-                )
-            ]
-        )
-
-    def run(self):
-        # Can only run this test if we have a Dataset object
-        if not isinstance(self.inputs.dataset, VMDataset):
-            raise ValueError("ClassImbalance requires a validmind Dataset object")
-
-        if self.inputs.dataset.target_column is None:
-            print("Skipping class_imbalance test because no target column is defined")
-            return
-
-        target_column = self.inputs.dataset.target_column
-        imbalance_percentages = self.inputs.dataset.df[target_column].value_counts(
-            normalize=True
+    if not dataset.target_column:
+        raise SkipTestError("No target column provided")
+
+    imbalance_percentages = dataset.df[dataset.target_column].value_counts(
+        normalize=True
+    )
+    if len(imbalance_percentages) > 10:
+        raise SkipTestError("Skipping target column with more than 10 classes")
+
+    classes = list(imbalance_percentages.index)
+
+    imbalanced_classes = []
+    for i, percentage in enumerate(imbalance_percentages.values):
+        proportion = percentage * 100
+        imbalanced_classes.append(
+            {
+                dataset.target_column: classes[i],
+                "Percentage of Rows (%)": f"{proportion:.2f}%",
+                "Pass/Fail": "Pass" if proportion > min_percent_threshold else "Fail",
+            }
         )
-        if len(imbalance_percentages) > 10:
-            raise SkipTestError(
-                f"Skipping {self.__class__.__name__} test as"
-                "target column as more than 10 classes"
-            )
-
-        classes = list(imbalance_percentages.index)
-        percentages = list(imbalance_percentages.values)
-
-        # Checking class imbalance
-        imbalanced_classes = []
-        for i, percentage in enumerate(percentages):
-            class_label = classes[i]
-            proportion = percentage * 100
-            passed = proportion > self.params["min_percent_threshold"]
-
-            imbalanced_classes.append(
-                {
-                    target_column: class_label,
-                    "Percentage of Rows (%)	": f"{proportion:.2f}%",
-                    "Pass/Fail": "Pass" if passed else "Fail",
-                }
-            )
-
-        resultset = pd.DataFrame(imbalanced_classes)
-        tests_failed = all(resultset["Pass/Fail"] == "Pass")
-        results = [
-            ThresholdTestResult(
-                column=target_column,
-                passed=tests_failed,
-                values=resultset.to_dict(orient="records"),
-            )
-        ]
-
-        # Create a bar chart trace
-        trace = go.Bar(
-            x=imbalance_percentages.index,
-            y=imbalance_percentages.values,
-        )
-
-        # Create a layout for the chart
-        layout = go.Layout(
-            title=f"Class Imbalance Results for Target Column {self.inputs.dataset.target_column}",
-            xaxis=dict(title="Class"),
-            yaxis=dict(title="Percentage"),
-        )
-
-        # Create a figure and add the trace and layout
-        fig = go.Figure(data=[trace], layout=layout)
-
-        return self.cache_results(
-            results,
-            passed=tests_failed,
-            figures=[
-                Figure(
-                    for_object=self,
-                    key=f"{self.name}",
-                    figure=fig,
-                )
-            ],
-        )
-
-    def test(self):
-        """Unit test for ClassImbalance"""
-        assert self.result is not None
-
-        assert self.result.test_results is not None
-        assert self.result.test_results.passed
 
-        assert self.result.figures is not None
+    trace = go.Bar(
+        x=imbalance_percentages.index,
+        y=imbalance_percentages.values,
+    )
+
+    layout = go.Layout(
+        title=f"{dataset.target_column} Class Imbalance",
+        xaxis=dict(title="Class"),
+        yaxis=dict(title="Percentage"),
+    )
+
+    return (
+        {
+            f"{dataset.target_column} Class Imbalance": imbalanced_classes,
+        },
+        go.Figure(data=[trace], layout=layout),
+        all(row["Pass/Fail"] == "Pass" for row in imbalanced_classes),
+    )

validmind/tests/data_validation/DatasetDescription.py

@@ -4,15 +4,15 @@
 
 import re
 from collections import Counter
-from dataclasses import dataclass
 
 import numpy as np
 from ydata_profiling.config import Settings
 from ydata_profiling.model.typeset import ProfilingTypeSet
 
+from validmind import tags, tasks
 from validmind.errors import UnsupportedColumnTypeError
 from validmind.logging import get_logger
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
+from validmind.vm_models import VMDataset
 
 DEFAULT_HISTOGRAM_BINS = 10
 DEFAULT_HISTOGRAM_BIN_SIZES = [5, 10, 20, 50]
@@ -20,37 +20,179 @@ DEFAULT_HISTOGRAM_BIN_SIZES = [5, 10, 20, 50]
 logger = get_logger(__name__)
 
 
-@dataclass
-class DatasetDescription(Metric):
+def infer_datatypes(df):
+    column_type_mappings = {}
+    typeset = ProfilingTypeSet(Settings())
+    variable_types = typeset.infer_type(df)
+
+    for column, type in variable_types.items():
+        if str(type) == "Unsupported":
+            if df[column].isnull().all():
+                column_type_mappings[column] = {"id": column, "type": "Null"}
+            else:
+                raise UnsupportedColumnTypeError(
+                    f"Unsupported type for column {column}. Please review all values in this dataset column."
+                )
+        else:
+            column_type_mappings[column] = {"id": column, "type": str(type)}
+
+    return list(column_type_mappings.values())
+
+
+def get_numerical_histograms(df, column):
+    """
+    Returns a collection of histograms for a numerical column, each one
+    with a different bin size
     """
-    Provides comprehensive analysis and statistical summaries of each field in a machine learning model's dataset.
+    values = df[column].to_numpy()
+    values_cleaned = values[~np.isnan(values)]
+
+    # bins='sturges'. Cannot use 'auto' until we review and fix its performance
+    #  on datasets with too many unique values
+    #
+    # 'sturges': R’s default method, only accounts for data size. Only optimal
+    # for gaussian data and underestimates number of bins for large non-gaussian datasets.
+    default_hist = np.histogram(values_cleaned, bins="sturges")
+
+    histograms = {
+        "default": {
+            "bin_size": len(default_hist[0]),
+            "histogram": {
+                "bin_edges": default_hist[1].tolist(),
+                "counts": default_hist[0].tolist(),
+            },
+        }
+    }
+
+    for bin_size in DEFAULT_HISTOGRAM_BIN_SIZES:
+        hist = np.histogram(values_cleaned, bins=bin_size)
+        histograms[f"bins_{bin_size}"] = {
+            "bin_size": bin_size,
+            "histogram": {
+                "bin_edges": hist[1].tolist(),
+                "counts": hist[0].tolist(),
+            },
+        }
+
+    return histograms
+
+
+def get_column_histograms(df, column, type_):
+    """
+    Returns a collection of histograms for a numerical or categorical column.
+    We store different combinations of bin sizes to allow analyzing the data better
+
+    Will be used in favor of _get_histogram in the future
+    """
+    # Set the minimum number of bins to nunique if it's less than the default
+    if type_ == "Numeric":
+        return get_numerical_histograms(df, column)
+    elif type_ == "Categorical" or type_ == "Boolean":
+        value_counts = df[column].value_counts()
+        return {
+            "default": {
+                "bin_size": len(value_counts),
+                "histogram": value_counts.to_dict(),
+            }
+        }
+    elif type_ == "Text":
+        # Combine all the text in the specified column
+        text_data = " ".join(df[column].astype(str))
+        # Split the text into words (tokens) using a regular expression
+        words = re.findall(r"\w+", text_data)
+        # Use Counter to count the frequency of each word
+        word_counts = Counter(words)
+
+        return {
+            "default": {
+                "bin_size": len(word_counts),
+                "histogram": dict(word_counts),
+            }
+        }
+    elif type_ == "Null":
+        logger.info(f"Ignoring histogram generation for null column {column}")
+    else:
+        raise ValueError(
+            f"Unsupported column type found when computing its histogram: {type_}"
+        )
+
+
+def describe_column(df, column):
+    """
+    Gets descriptive statistics for a single column in a Pandas DataFrame.
+    """
+    column_type = column["type"]
+
+    # Initialize statistics with count for all column types
+    column["statistics"] = {
+        "count": df[column["id"]].count(),
+        "n_missing": df[column["id"]].isna().sum(),
+        "missing": df[column["id"]].isna().sum() / len(df[column["id"]]),
+        "n_distinct": df[column["id"]].nunique(),
+        "distinct": df[column["id"]].nunique() / len(df[column["id"]]),
+    }
+
+    # Boolean (binary) columns should be reported as categorical
+    if column_type == "Boolean" or df[column["id"]].nunique() == 2:
+        column["type"] = "Categorical"  # Change the type to Categorical
+        top_value = df[column["id"]].value_counts().nlargest(1)
+        column["statistics"].update(
+            {
+                "unique": df[column["id"]].nunique(),
+                "top": top_value.index[0],
+                "freq": top_value.values[0],
+            }
+        )
+    elif column_type == "Numeric":
+        column["statistics"].update(
+            df[column["id"]]
+            .describe(percentiles=[0.25, 0.5, 0.75, 0.9, 0.95])
+            .to_dict()
+        )
+    elif column_type == "Categorical" or column_type == "Text":
+        column["statistics"].update(
+            df[column["id"]].astype("category").describe().to_dict()
+        )
+
+    column["histograms"] = get_column_histograms(
+        df, column["id"], column["type"]
+    )  # Use updated type
+
+    return column
+
+
+@tags("tabular_data", "time_series_data", "text_data")
+@tasks("classification", "regression", "text_classification", "text_summarization")
+def DatasetDescription(dataset: VMDataset):
+    """
+    Provides comprehensive analysis and statistical summaries of each column in a machine learning model's dataset.
 
     ### Purpose
 
     The test depicted in the script is meant to run a comprehensive analysis on a Machine Learning model's datasets.
-    The test or metric is implemented to obtain a complete summary of the fields in the dataset, including vital
-    statistics of each field such as count, distinct values, missing values, histograms for numerical, categorical,
-    boolean, and text fields. This summary gives a comprehensive overview of the dataset to better understand the
+    The test or metric is implemented to obtain a complete summary of the columns in the dataset, including vital
+    statistics of each column such as count, distinct values, missing values, histograms for numerical, categorical,
+    boolean, and text columns. This summary gives a comprehensive overview of the dataset to better understand the
     characteristics of the data that the model is trained on or evaluates.
 
     ### Test Mechanism
 
     The DatasetDescription class accomplishes the purpose as follows: firstly, the test method "run" infers the data
-    type of each column in the dataset and stores the details (id, column type). For each field, the
-    "describe_dataset_field" method is invoked to collect statistical information about the field, including count,
+    type of each column in the dataset and stores the details (id, column type). For each column, the
+    "describe_column" method is invoked to collect statistical information about the column, including count,
     missing value count and its proportion to the total, unique value count, and its proportion to the total. Depending
-    on the data type of a field, histograms are generated that reflect the distribution of data within the field.
-    Numerical fields use the "get_numerical_histograms" method to calculate histogram distribution, whereas for
-    categorical, boolean and text fields, a histogram is computed with frequencies of each unique value in the
+    on the data type of a column, histograms are generated that reflect the distribution of data within the column.
+    Numerical columns use the "get_numerical_histograms" method to calculate histogram distribution, whereas for
+    categorical, boolean and text columns, a histogram is computed with frequencies of each unique value in the
     datasets. For unsupported types, an error is raised. Lastly, a summary table is built to aggregate all the
-    statistical insights and histograms of the fields in a dataset.
+    statistical insights and histograms of the columns in a dataset.
 
     ### Signs of High Risk
 
-    - High ratio of missing values to total values in one or more fields which may impact the quality of the
+    - High ratio of missing values to total values in one or more columns which may impact the quality of the
     predictions.
-    - Unsupported data types in dataset fields.
-    - Large number of unique values in the dataset's fields which might make it harder for the model to establish
+    - Unsupported data types in dataset columns.
+    - Large number of unique values in the dataset's columns which might make it harder for the model to establish
     patterns.
     - Extreme skewness or irregular distribution of data as reflected in the histograms.
 
@@ -65,201 +207,30 @@ class DatasetDescription(Metric):
 
     ### Limitations
 
-    - The computation can be expensive from a resource standpoint, particularly for large datasets with numerous fields.
+    - The computation can be expensive from a resource standpoint, particularly for large datasets with numerous columns.
     - The histograms use an arbitrary number of bins which may not be the optimal number of bins for specific data
     distribution.
     - Unsupported data types for columns will raise an error which may limit evaluating the dataset.
-    - Fields with all null or missing values are not included in histogram computation.
+    - Columns with all null or missing values are not included in histogram computation.
     - This test only validates the quality of the dataset but doesn't address the model's performance directly.
     """
-
-    name = "dataset_description"
-    required_inputs = ["dataset"]
-    tasks = [
-        "classification",
-        "regression",
-        "text_classification",
-        "text_summarization",
-    ]
-    tags = ["tabular_data", "time_series_data", "text_data"]
-
-    def summary(self, metric_value):
-        """
-        Build a dataset summary table. metric_value is a list of fields where each field
-        has an id, type (Numeric or Categorical), and statistics. The statistics object
-        depends on the type being Numeric or Categorical. For Numeric fields, it has
-        the following keys: count, mean, std, min, 25%, 50%, 75%, 90%, 95%, max. For
-        categorical fields, it has the following keys: count, unique, top, freq.
-        """
-        results_table = []
-        for field in metric_value:
-            field_id = field["id"]
-            field_type = field["type"]
-            field_statistics = field["statistics"]
-
-            results_table.append(
-                {
-                    "Name": field_id,
-                    "Type": field_type,
-                    "Count": field_statistics["count"],
-                    "Missing": field_statistics["n_missing"],
-                    "Missing %": field_statistics["missing"],
-                    "Distinct": field_statistics["n_distinct"],
-                    "Distinct %": field_statistics["distinct"],
-                }
-            )
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=results_table,
-                    metadata=ResultTableMetadata(title="Dataset Description"),
-                )
-            ]
-        )
-
-    def run(self):
-        results = []
-        for ds_field in self.infer_datatype(self.inputs.dataset.df):
-            self.describe_dataset_field(self.inputs.dataset.df, ds_field)
-            results.append(ds_field)
-        return self.cache_results(results)
-
-    def infer_datatype(self, df):
-        vm_dataset_variables = {}
-        typeset = ProfilingTypeSet(Settings())
-        variable_types = typeset.infer_type(df)
-
-        for column, type in variable_types.items():
-            if str(type) == "Unsupported":
-                if df[column].isnull().all():
-                    vm_dataset_variables[column] = {"id": column, "type": "Null"}
-                else:
-                    raise UnsupportedColumnTypeError(
-                        f"Unsupported type for column {column}. Please review all values in this dataset column."
-                    )
-            else:
-                vm_dataset_variables[column] = {"id": column, "type": str(type)}
-
-        return list(vm_dataset_variables.values())
-
-    def describe_dataset_field(self, df, field):
-        """
-        Gets descriptive statistics for a single field in a Pandas DataFrame.
-        """
-        field_type = field["type"]
-
-        # - When we call describe on one field at a time, Pandas will
-        #   know better if it needs to report on numerical or categorical statistics
-        # - Boolean (binary) fields should be reported as categorical
-        #       (force to categorical when nunique == 2)
-        if field_type == ["Boolean"] or df[field["id"]].nunique() == 2:
-            top_value = df[field["id"]].value_counts().nlargest(1)
-
-            field["statistics"] = {
-                "count": df[field["id"]].count(),
-                "unique": df[field["id"]].nunique(),
-                "top": top_value.index[0],
-                "freq": top_value.values[0],
-            }
-        elif field_type == "Numeric":
-            field["statistics"] = (
-                df[field["id"]]
-                .describe(percentiles=[0.25, 0.5, 0.75, 0.9, 0.95])
-                .to_dict()
-            )
-        elif field_type == "Categorical" or field_type == "Text":
-            field["statistics"] = (
-                df[field["id"]].astype("category").describe().to_dict()
-            )
-
-        # Initialize statistics object for non-numeric or categorical fields
-        if "statistics" not in field:
-            field["statistics"] = {}
-
-        field["statistics"]["n_missing"] = df[field["id"]].isna().sum()
-        field["statistics"]["missing"] = field["statistics"]["n_missing"] / len(
-            df[field["id"]]
-        )
-        field["statistics"]["n_distinct"] = df[field["id"]].nunique()
-        field["statistics"]["distinct"] = field["statistics"]["n_distinct"] / len(
-            df[field["id"]]
-        )
-
-        field["histograms"] = self.get_field_histograms(df, field["id"], field_type)
-
-    def get_field_histograms(self, df, field, type_):
-        """
-        Returns a collection of histograms for a numerical or categorical field.
-        We store different combinations of bin sizes to allow analyzing the data better
-
-        Will be used in favor of _get_histogram in the future
-        """
-        # Set the minimum number of bins to nunique if it's less than the default
-        if type_ == "Numeric":
-            return self.get_numerical_histograms(df, field)
-        elif type_ == "Categorical" or type_ == "Boolean":
-            value_counts = df[field].value_counts()
-            return {
-                "default": {
-                    "bin_size": len(value_counts),
-                    "histogram": value_counts.to_dict(),
-                }
-            }
-        elif type_ == "Text":
-            # Combine all the text in the specified field
-            text_data = " ".join(df[field].astype(str))
-            # Split the text into words (tokens) using a regular expression
-            words = re.findall(r"\w+", text_data)
-            # Use Counter to count the frequency of each word
-            word_counts = Counter(words)
-
-            return {
-                "default": {
-                    "bin_size": len(word_counts),
-                    "histogram": dict(word_counts),
-                }
+    df = dataset.df
+
+    results = []
+    for column in infer_datatypes(df):
+        results.append(describe_column(df, column))
+
+    return {
+        "Dataset Description": [
+            {
+                "Name": column["id"],
+                "Type": column["type"],
+                "Count": column["statistics"]["count"],
+                "Missing": column["statistics"]["n_missing"],
+                "Missing %": column["statistics"]["missing"],
+                "Distinct": column["statistics"]["n_distinct"],
+                "Distinct %": column["statistics"]["distinct"],
             }
-        elif type_ == "Null":
-            logger.info(f"Ignoring histogram generation for null column {field}")
-        else:
-            raise ValueError(
-                f"Unsupported field type found when computing its histogram: {type_}"
-            )
-
-    def get_numerical_histograms(self, df, field):
-        """
-        Returns a collection of histograms for a numerical field, each one
-        with a different bin size
-        """
-        values = df[field].to_numpy()
-        values_cleaned = values[~np.isnan(values)]
-
-        # bins='sturges'. Cannot use 'auto' until we review and fix its performance
-        #  on datasets with too many unique values
-        #
-        # 'sturges': R’s default method, only accounts for data size. Only optimal
-        # for gaussian data and underestimates number of bins for large non-gaussian datasets.
-        default_hist = np.histogram(values_cleaned, bins="sturges")
-
-        histograms = {
-            "default": {
-                "bin_size": len(default_hist[0]),
-                "histogram": {
-                    "bin_edges": default_hist[1].tolist(),
-                    "counts": default_hist[0].tolist(),
-                },
-            }
-        }
-
-        for bin_size in DEFAULT_HISTOGRAM_BIN_SIZES:
-            hist = np.histogram(values_cleaned, bins=bin_size)
-            histograms[f"bins_{bin_size}"] = {
-                "bin_size": bin_size,
-                "histogram": {
-                    "bin_edges": hist[1].tolist(),
-                    "counts": hist[0].tolist(),
-                },
-            }
-
-        return histograms
+            for column in results
+        ]
+    }