validmind 2.5.25__py3-none-any.whl → 2.6.8__py3-none-any.whl

validmind/tests/data_validation/DatasetSplit.py

@@ -2,10 +2,22 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from validmind.vm_models import Metric, ResultSummary, ResultTable
+from typing import List
 
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset
 
-class DatasetSplit(Metric):
+DATASET_LABELS = {
+    "train_ds": "Training",
+    "test_ds": "Test",
+    "validation_ds": "Validation",
+    "total": "Total",
+}
+
+
+@tags("tabular_data", "time_series_data", "text_data")
+@tasks("classification", "regression", "text_classification", "text_summarization")
+def DatasetSplit(datasets: List[VMDataset]):
     """
     Evaluates and visualizes the distribution proportions among training, testing, and validation datasets of an ML
     model.
@@ -47,80 +59,43 @@ class DatasetSplit(Metric):
     - Potential lack of compatibility with more complex modes of data splitting (for example, stratified or time-based
     splits) could limit the applicability of this test.
     """
-
-    name = "dataset_split"
-    required_inputs = ["datasets"]
-    tasks = [
-        "classification",
-        "regression",
-        "text_classification",
-        "text_summarization",
-    ]
-
-    tags = ["tabular_data", "time_series_data", "text_data"]
-
-    dataset_labels = {
-        "train_ds": "Training",
-        "test_ds": "Test",
-        "validation_ds": "Validation",
-        "total": "Total",
-    }
-
-    def summary(self, raw_results):
-        """
-        Returns a summarized representation of the dataset split information
-        """
-        table_records = []
-        for key, value in raw_results.items():
-            if key.endswith("_size"):
-                dataset_name = key.replace("_size", "")
-                if dataset_name == "total":
-                    table_records.append(
-                        {
-                            "Dataset": "Total",
-                            "Size": value,
-                            "Proportion": "100%",
-                        }
-                    )
-                    continue
-
-                proportion = raw_results[f"{dataset_name}_proportion"] * 100
-                table_records.append(
+    results = {}
+    total_size = 0
+
+    # First calculate the total size of the dataset
+    for dataset in datasets:
+        if dataset is not None:
+            total_size += len(dataset.df)
+
+    # Then calculate the proportion of each dataset
+    for dataset in datasets:
+        if dataset is not None:
+            results[f"{dataset.input_id}_size"] = len(dataset.df)
+            results[f"{dataset.input_id}_proportion"] = len(dataset.df) / total_size
+
+    results["total_size"] = total_size
+
+    table = []
+    for key, value in results.items():
+        if key.endswith("_size"):
+            dataset_name = key.replace("_size", "")
+            if dataset_name == "total":
+                table.append(
                     {
-                        "Dataset": dataset_name,  # DatasetSplit.dataset_labels[dataset_name],
+                        "Dataset": "Total",
                         "Size": value,
-                        "Proportion": f"{proportion:.2f}%",
+                        "Proportion": "100%",
                     }
                 )
-
-        return ResultSummary(results=[ResultTable(data=table_records)])
-
-    def run(self):
-        # Try to extract metrics from each available dataset
-        available_datasets = self.inputs.datasets
-        results = {}
-        total_size = 0
-
-        # First calculate the total size of the dataset
-        for dataset in available_datasets:
-            if dataset is not None:
-                total_size += len(dataset.df)
-
-        # Then calculate the proportion of each dataset
-        for dataset in available_datasets:
-            if dataset is not None:
-                results[f"{dataset.input_id}_size"] = len(dataset.df)
-                results[f"{dataset.input_id}_proportion"] = len(dataset.df) / total_size
-
-        results["total_size"] = total_size
-
-        return self.cache_results(results)
-
-    def test(self):
-        """Unit Test for DatasetSplit Metric"""
-        assert self.result is not None
-
-        assert self.result.metric is not None
-        assert isinstance(self.result.metric.value, dict)
-
-        assert self.result.figures is None
+                continue
+
+            proportion = results[f"{dataset_name}_proportion"] * 100
+            table.append(
+                {
+                    "Dataset": dataset_name,  # DatasetSplit.dataset_labels[dataset_name],
+                    "Size": value,
+                    "Proportion": f"{proportion:.2f}%",
+                }
+            )
+
+    return table

validmind/tests/data_validation/DescriptiveStatistics.py

@@ -2,16 +2,51 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import pandas as pd
 
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
 from validmind.utils import format_records
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
+from validmind.vm_models import VMDataset
+
+
+def get_summary_statistics_numerical(df, numerical_fields):
+    percentiles = [0.25, 0.5, 0.75, 0.90, 0.95]
+
+    summary_stats = df[numerical_fields].describe(percentiles=percentiles).T
+    summary_stats = summary_stats[
+        ["count", "mean", "std", "min", "25%", "50%", "75%", "90%", "95%", "max"]
+    ]
+    summary_stats.columns = summary_stats.columns.str.title()
+    summary_stats.reset_index(inplace=True)
+    summary_stats.rename(columns={"index": "Name"}, inplace=True)
+
+    return summary_stats
+
+
+def get_summary_statistics_categorical(df, categorical_fields):
+    summary_stats = pd.DataFrame()
 
+    for column in df[categorical_fields].columns:
+        top_value = df[column].value_counts().idxmax()
+        top_freq = df[column].value_counts().max()
+        summary_stats.loc[column, "Count"] = df[column].count()
+        summary_stats.loc[column, "Number of Unique Values"] = df[column].nunique()
+        summary_stats.loc[column, "Top Value"] = top_value
+        summary_stats.loc[column, "Top Value Frequency"] = top_freq
+        summary_stats.loc[column, "Top Value Frequency %"] = (
+            top_freq / df[column].count()
+        ) * 100
 
-@dataclass
-class DescriptiveStatistics(Metric):
+    summary_stats.reset_index(inplace=True)
+    summary_stats.rename(columns={"index": "Name"}, inplace=True)
+
+    return summary_stats
+
+
+@tags("tabular_data", "time_series_data")
+@tasks("classification", "regression")
+def DescriptiveStatistics(dataset: VMDataset):
     """
     Performs a detailed descriptive statistical analysis of both numerical and categorical data within a model's
     dataset.
@@ -57,84 +92,23 @@ class DescriptiveStatistics(Metric):
     - Should be used in conjunction with other statistical tests to provide a comprehensive understanding of the
     model's data.
     """
-
-    name = "descriptive_statistics"
-    required_inputs = ["dataset"]
-    tasks = ["classification", "regression"]
-    tags = ["tabular_data", "time_series_data"]
-
-    def get_summary_statistics_numerical(self, df, numerical_fields):
-        percentiles = [0.25, 0.5, 0.75, 0.90, 0.95]
-
-        summary_stats = df[numerical_fields].describe(percentiles=percentiles).T
-        summary_stats = summary_stats[
-            ["count", "mean", "std", "min", "25%", "50%", "75%", "90%", "95%", "max"]
-        ]
-        summary_stats.columns = summary_stats.columns.str.title()
-        summary_stats.reset_index(inplace=True)
-        summary_stats.rename(columns={"index": "Name"}, inplace=True)
-
-        return format_records(summary_stats)
-
-    def get_summary_statistics_categorical(self, df, categorical_fields):
-        summary_stats = pd.DataFrame()
-
-        for column in df[categorical_fields].columns:
-            top_value = df[column].value_counts().idxmax()
-            top_freq = df[column].value_counts().max()
-            summary_stats.loc[column, "Count"] = df[column].count()
-            summary_stats.loc[column, "Number of Unique Values"] = df[column].nunique()
-            summary_stats.loc[column, "Top Value"] = top_value
-            summary_stats.loc[column, "Top Value Frequency"] = top_freq
-            summary_stats.loc[column, "Top Value Frequency %"] = (
-                top_freq / df[column].count()
-            ) * 100
-
-        summary_stats.reset_index(inplace=True)
-        summary_stats.rename(columns={"index": "Name"}, inplace=True)
-
-        return format_records(summary_stats)
-
-    def summary(self, metric_value):
-        """
-        Build two tables: one for summarizing numerical variables and one for categorical variables
-        """
-        summary_stats_numerical = metric_value["numerical"]
-        summary_stats_categorical = metric_value["categorical"]
-        results = []
-        if len(summary_stats_numerical) != 0:
-            results.append(
-                ResultTable(
-                    data=summary_stats_numerical,
-                    metadata=ResultTableMetadata(title="Numerical Variables"),
-                )
-            )
-        if len(summary_stats_categorical) != 0:
-            results.append(
-                ResultTable(
-                    data=summary_stats_categorical,
-                    metadata=ResultTableMetadata(title="Categorical Variables"),
-                )
-            )
-
-        return ResultSummary(results=results)
-
-    def run(self):
-        feature_columns = self.inputs.dataset.feature_columns
-        numerical_feature_columns = self.inputs.dataset.feature_columns_numeric
-        categorical_feature_columns = self.inputs.dataset.feature_columns_categorical
-
-        df = self.inputs.dataset.df[feature_columns]
-
-        summary_stats_numerical = self.get_summary_statistics_numerical(
-            df, numerical_feature_columns
-        )
-        summary_stats_categorical = self.get_summary_statistics_categorical(
-            df, categorical_feature_columns
-        )
-        return self.cache_results(
-            {
-                "numerical": summary_stats_numerical,
-                "categorical": summary_stats_categorical,
-            }
+    tables = {}
+
+    summary_stats_numerical = get_summary_statistics_numerical(
+        dataset.df, dataset.feature_columns_numeric
+    )
+    if not summary_stats_numerical.empty:
+        tables["Numerical Variables"] = format_records(summary_stats_numerical)
+
+    summary_stats_categorical = get_summary_statistics_categorical(
+        dataset.df, dataset.feature_columns_categorical
+    )
+    if not summary_stats_categorical.empty:
+        tables["Categorical Variables"] = format_records(summary_stats_categorical)
+
+    if not tables:
+        raise SkipTestError(
+            "No numerical or categorical variables found in the dataset."
         )
+
+    return tables

validmind/tests/data_validation/{DFGLSArch.py → DickeyFullerGLS.py}

@@ -2,20 +2,21 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import pandas as pd
 from arch.unitroot import DFGLS
 from numpy.linalg import LinAlgError
 
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
 from validmind.logging import get_logger
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
+from validmind.vm_models import VMDataset
 
 logger = get_logger(__name__)
 
 
-@dataclass
-class DFGLSArch(Metric):
+@tags("time_series_data", "forecasting", "unit_root_test")
+@tasks("regression")
+def DickeyFullerGLS(dataset: VMDataset):
     """
     Assesses stationarity in time series data using the Dickey-Fuller GLS test to determine the order of integration.
 
@@ -56,77 +57,44 @@ class DFGLSArch(Metric):
     - The test also presents challenges when dealing with shorter time series data or volatile data, not producing
     reliable results in these cases.
     """
+    df = dataset.df.dropna()
 
-    name = "dickey_fuller_gls"
-    required_inputs = ["dataset"]
-    tasks = ["regression"]
-    tags = ["time_series_data", "forecasting", "unit_root_test"]
-
-    def run(self):
-        """
-        Calculates Dickey-Fuller GLS metric for each of the dataset features
-        """
-        dataset = self.inputs.dataset.df
-
-        # Check if the dataset is a time series
-        if not isinstance(dataset.index, (pd.DatetimeIndex, pd.PeriodIndex)):
-            raise ValueError(
-                "Dataset index must be a datetime or period index for time series analysis."
-            )
+    if not isinstance(df.index, (pd.DatetimeIndex, pd.PeriodIndex)):
+        raise SkipTestError(
+            "Dataset index must be a datetime or period index for time series analysis."
+        )
 
-        # Preprocessing: Drop rows with any NaN values
-        if dataset.isnull().values.any():
-            logger.warning(
-                "Dataset contains missing values. Rows with NaNs will be dropped."
+    df = df.apply(pd.to_numeric, errors="coerce")
+
+    dfgls_values = []
+
+    for col in df.columns:
+        try:
+            dfgls_out = DFGLS(df[col].values)
+            dfgls_values.append(
+                {
+                    "Variable": col,
+                    "stat": dfgls_out.stat,
+                    "pvalue": dfgls_out.pvalue,
+                    "usedlag": dfgls_out.lags,
+                    "nobs": dfgls_out.nobs,
+                }
             )
-            dataset = dataset.dropna()
-
-        # Convert to numeric and handle non-numeric data
-        dataset = dataset.apply(pd.to_numeric, errors="coerce")
-
-        # Initialize a list to store DFGLS results
-        dfgls_values = []
-
-        for col in dataset.columns:
-            try:
-                dfgls_out = DFGLS(dataset[col].values)
-                dfgls_values.append(
-                    {
-                        "Variable": col,
-                        "stat": dfgls_out.stat,
-                        "pvalue": dfgls_out.pvalue,
-                        "usedlag": dfgls_out.lags,
-                        "nobs": dfgls_out.nobs,
-                    }
-                )
-            except LinAlgError as e:
-                logger.error(
-                    f"SVD did not converge while processing column '{col}'. This could be due to numerical instability or multicollinearity. Error details: {e}"
-                )
-                dfgls_values.append(
-                    {
-                        "Variable": col,
-                        "stat": None,
-                        "pvalue": None,
-                        "usedlag": None,
-                        "nobs": None,
-                        "error": str(e),
-                    }
-                )
-
-        return self.cache_results({"dfgls_results": dfgls_values})
-
-    def summary(self, metric_value):
-        """
-        Build a table for summarizing the DFGLS results
-        """
-        dfgls_results = metric_value["dfgls_results"]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=dfgls_results,
-                    metadata=ResultTableMetadata(title="DFGLS Test Results"),
-                )
-            ]
-        )
+        except LinAlgError as e:
+            logger.error(
+                f"SVD did not converge while processing column '{col}'. This could be due to numerical instability or multicollinearity. Error details: {e}"
+            )
+            dfgls_values.append(
+                {
+                    "Variable": col,
+                    "stat": None,
+                    "pvalue": None,
+                    "usedlag": None,
+                    "nobs": None,
+                    "error": str(e),
+                }
+            )
+
+    return {
+        "DFGLS Test Results": dfgls_values,
+    }

validmind/tests/data_validation/Duplicates.py

@@ -2,22 +2,14 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-from typing import List
-
 import pandas as pd
 
-from validmind.vm_models import (
-    ResultSummary,
-    ResultTable,
-    ResultTableMetadata,
-    ThresholdTest,
-    ThresholdTestResult,
-)
+from validmind import tags, tasks
 
 
-@dataclass
-class Duplicates(ThresholdTest):
+@tags("tabular_data", "data_quality", "text_data")
+@tasks("classification", "regression")
+def Duplicates(dataset, min_threshold=1):
     """
     Tests dataset for duplicate entries, ensuring model reliability via data quality verification.
 
@@ -59,82 +51,21 @@ class Duplicates(ThresholdTest):
     for very large datasets.
     - Can only check for exact duplicates and may miss semantically similar information packaged differently.
     """
+    df = dataset.df[dataset.text_column or dataset.feature_columns]
+
+    duplicate_rows_count = df.duplicated().sum()
+    percentage_duplicate_rows = (duplicate_rows_count / len(df)) * 100
+
+    result_df = pd.DataFrame(
+        {
+            "Number of Duplicates": [duplicate_rows_count],
+            "Percentage of Rows (%)": [percentage_duplicate_rows],
+        }
+    )
+
+    # test has passed if the total sum of duplicates is less than the threshold
+    passed = result_df["Number of Duplicates"].sum() < min_threshold
 
-    name = "duplicates"
-    required_inputs = ["dataset"]
-    default_params = {"min_threshold": 1}
-    tasks = ["classification", "regression"]
-    tags = ["tabular_data", "data_quality", "text_data"]
-
-    def summary(self, results: List[ThresholdTestResult], all_passed: bool):
-        """
-        The duplicates test returns results like these:
-        [{"values": {"n_duplicates": 0, "p_duplicates": 0.0}, "passed": true}]
-        So we build a table with 1 row and show number of duplicates and percentage of duplicates.
-        """
-        result = results[0]
-        results_table = [{k: v for k, v in row.items()} for row in result.values]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=results_table,
-                    metadata=ResultTableMetadata(
-                        title="Duplicate Rows Results for Dataset"
-                    ),
-                )
-            ]
-        )
-
-    def run(self):
-        if self.inputs.dataset.text_column:
-            columns = self.inputs.dataset.text_column
-        else:
-            columns = self.inputs.dataset.feature_columns
-
-        df = self.inputs.dataset.df[columns]
-        # Find duplicate rows
-        duplicate_rows = df.duplicated()
-
-        # Calculate number of duplicate rows
-        duplicate_rows_count = duplicate_rows.sum()
-
-        # Calculate total number of rows
-        total_rows = len(df)
-
-        # Calculate percentage of duplicate rows
-        percentage_duplicate_rows = (duplicate_rows_count / total_rows) * 100
-
-        # Create a DataFrame with results
-        result_df = pd.DataFrame(
-            {
-                "Number of Duplicates": [duplicate_rows_count],
-                "Percentage of Rows (%)": [percentage_duplicate_rows],
-            }
-        )
-
-        # test has passed if the total sum of duplicates is less than the threshold
-        n_duplicates = result_df["Number of Duplicates"].sum()
-        passed = n_duplicates < self.params["min_threshold"]
-
-        results = [
-            ThresholdTestResult(
-                passed=passed,
-                values=result_df.to_dict(orient="records"),
-            )
-        ]
-
-        return self.cache_results(results, passed=all([r.passed for r in results]))
-
-    def test(self):
-        # Check that result object is not None
-        assert self.result is not None
-        # Check that we have a list of test results
-        assert isinstance(self.result.test_results.results, list)
-        # Check if the 'passed' variable in results reflects the test correctly
-        for result in self.result.test_results.results[1:]:
-            assert result.passed == (
-                result.values["n_duplicates"] < self.params["min_threshold"]
-            )
-        expected_results_count = 1
-        assert len(self.result.test_results.results) == expected_results_count
+    return {
+        "Duplicate Rows Results for Dataset": result_df.to_dict(orient="records")
+    }, passed