validmind 2.3.5__py3-none-any.whl → 2.4.1__py3-none-any.whl

validmind/tests/data_validation/Skewness.py

@@ -62,10 +62,8 @@ class Skewness(ThresholdTest):
     name = "skewness"
     required_inputs = ["dataset"]
     default_params = {"max_threshold": 1}
-    metadata = {
-        "task_types": ["classification", "regression"],
-        "tags": ["tabular_data", "data_quality"],
-    }
+    tasks = ["classification", "regression"]
+    tags = ["tabular_data", "data_quality"]
 
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         """

validmind/tests/data_validation/SpreadPlot.py

@@ -54,10 +54,8 @@ class SpreadPlot(Metric):
 
     name = "spread_plot"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data", "visualization"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data", "visualization"]
 
     @staticmethod
     def plot_spread(series1, series2, ax=None):

validmind/tests/data_validation/TabularCategoricalBarPlots.py

@@ -43,10 +43,8 @@ class TabularCategoricalBarPlots(Metric):
 
     name = "tabular_categorical_bar_plots"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["classification", "regression"],
-        "tags": ["tabular_data", "visualization"],
-    }
+    tasks = ["classification", "regression"]
+    tags = ["tabular_data", "visualization"]
 
     def run(self):
         df = self.inputs.dataset.df

validmind/tests/data_validation/TabularDateTimeHistograms.py

@@ -48,10 +48,8 @@ class TabularDateTimeHistograms(Metric):
     name = "tabular_datetime_histograms"
     required_inputs = ["dataset"]
 
-    metadata = {
-        "task_types": ["classification", "regression"],
-        "tags": ["time_series_data", "visualization"],
-    }
+    tasks = ["classification", "regression"]
+    tags = ["time_series_data", "visualization"]
 
     def run(self):
         df = self.inputs.dataset.df

validmind/tests/data_validation/TabularDescriptionTables.py

@@ -57,10 +57,8 @@ class TabularDescriptionTables(Metric):
     name = "tabular_description_tables"
     required_inputs = ["dataset"]
 
-    metadata = {
-        "task_types": ["classification", "regression"],
-        "tags": ["tabular_data"],
-    }
+    tasks = ["classification", "regression"]
+    tags = ["tabular_data"]
 
     def get_summary_statistics_numerical(self, numerical_fields):
         summary_stats = self.inputs.dataset.df[numerical_fields].describe().T

validmind/tests/data_validation/TabularNumericalHistograms.py

@@ -52,10 +52,8 @@ class TabularNumericalHistograms(Metric):
     name = "tabular_numerical_histograms"
     required_inputs = ["dataset"]
 
-    metadata = {
-        "task_types": ["classification", "regression"],
-        "tags": ["tabular_data", "visualization"],
-    }
+    tasks = ["classification", "regression"]
+    tags = ["tabular_data", "visualization"]
 
     def run(self):
         df = self.inputs.dataset.df

validmind/tests/data_validation/TargetRateBarPlots.py

@@ -47,10 +47,8 @@ class TargetRateBarPlots(Metric):
     name = "target_rate_bar_plots"
     required_inputs = ["dataset"]
     default_params = {"default_column": None, "columns": None}
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["tabular_data", "visualization", "categorical_data"],
-    }
+    tasks = ["classification"]
+    tags = ["tabular_data", "visualization", "categorical_data"]
 
     def plot_loan_default_ratio(self, default_column, columns=None):
         df = self.inputs.dataset.df

validmind/tests/data_validation/TimeSeriesFrequency.py

@@ -59,10 +59,8 @@ class TimeSeriesFrequency(ThresholdTest):
 
     name = "time_series_frequency"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data"]
 
     def summary(self, results, all_passed):
         """

validmind/tests/data_validation/TimeSeriesLinePlot.py

@@ -47,10 +47,8 @@ class TimeSeriesLinePlot(Metric):
 
     name = "time_series_line_plot"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data", "visualization"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data", "visualization"]
 
     def run(self):
         # Check if index is datetime

validmind/tests/data_validation/TimeSeriesMissingValues.py

@@ -65,10 +65,8 @@ class TimeSeriesMissingValues(ThresholdTest):
     name = "time_series_missing_values"
     required_inputs = ["dataset"]
     default_params = {"min_threshold": 1}
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data"]
 
     def summary(self, results, all_passed):
         results_table = [

validmind/tests/data_validation/TimeSeriesOutliers.py

@@ -62,10 +62,8 @@ class TimeSeriesOutliers(ThresholdTest):
     name = "time_series_outliers"
     required_inputs = ["dataset"]
     default_params = {"zscore_threshold": 3}
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data"]
 
     def summary(self, results, all_passed: bool):
         """

validmind/tests/data_validation/TooManyZeroValues.py

@@ -70,10 +70,8 @@ class TooManyZeroValues(ThresholdTest):
     required_inputs = ["dataset"]
     default_params = {"max_percent_threshold": 0.03}
 
-    metadata = {
-        "task_types": ["regression", "classification"],
-        "tags": ["tabular_data"],
-    }
+    tasks = ["regression", "classification"]
+    tags = ["tabular_data"]
 
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         """

validmind/tests/data_validation/UniqueRows.py

@@ -57,10 +57,8 @@ class UniqueRows(ThresholdTest):
     required_inputs = ["dataset"]
     default_params = {"min_percent_threshold": 1}
 
-    metadata = {
-        "task_types": ["regression", "classification"],
-        "tags": ["tabular_data"],
-    }
+    tasks = ["regression", "classification"]
+    tags = ["tabular_data"]
 
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         """

validmind/tests/data_validation/WOEBinPlots.py

@@ -60,10 +60,8 @@ class WOEBinPlots(Metric):
     name = "woe_bin_plots"
     required_inputs = ["dataset"]
     default_params = {"breaks_adj": None, "fig_height": 600, "fig_width": 500}
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["tabular_data", "visualization", "categorical_data"],
-    }
+    tasks = ["classification"]
+    tags = ["tabular_data", "visualization", "categorical_data"]
 
     def run(self):
         df = self.inputs.dataset.df

validmind/tests/data_validation/WOEBinTable.py

@@ -48,10 +48,8 @@ class WOEBinTable(Metric):
     name = "woe_bin_table"
     required_inputs = ["dataset"]
     default_params = {"breaks_adj": None}
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["tabular_data", "categorical_data"],
-    }
+    tasks = ["classification"]
+    tags = ["tabular_data", "categorical_data"]
 
     def run(self):
         target_column = self.inputs.dataset.target_column

validmind/tests/data_validation/ZivotAndrewsArch.py

@@ -51,10 +51,8 @@ class ZivotAndrewsArch(Metric):
 
     name = "zivot_andrews"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data", "stationarity", "unit_root_test"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data", "stationarity", "unit_root_test"]
 
     def run(self):
         """

validmind/tests/data_validation/nlp/CommonWords.py

@@ -53,10 +53,8 @@ class CommonWords(Metric):
 
     name = "common_words"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["nlp", "text_data", "visualization", "frequency_analysis"],
-    }
+    tasks = ["text_classification", "text_summarization"]
+    tags = ["nlp", "text_data", "visualization", "frequency_analysis"]
 
     def run(self):
         # Can only run this test if we have a Dataset object

validmind/tests/data_validation/nlp/Hashtags.py

@@ -56,10 +56,8 @@ class Hashtags(ThresholdTest):
     name = "hashtags"
     required_inputs = ["dataset"]
     default_params = {"top_hashtags": 25}
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["nlp", "text_data", "visualization", "frequency_analysis"],
-    }
+    tasks = ["text_classification", "text_summarization"]
+    tags = ["nlp", "text_data", "visualization", "frequency_analysis"]
 
     def run(self):
         # Can only run this test if we have a Dataset object

validmind/tests/data_validation/nlp/Mentions.py

@@ -56,10 +56,8 @@ class Mentions(ThresholdTest):
 
     required_inputs = ["dataset"]
     default_params = {"top_mentions": 25}
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["nlp", "text_data", "visualization", "frequency_analysis"],
-    }
+    tasks = ["text_classification", "text_summarization"]
+    tags = ["nlp", "text_data", "visualization", "frequency_analysis"]
 
     def run(self):
         # Can only run this test if we have a Dataset object

validmind/tests/data_validation/nlp/Punctuations.py

@@ -52,10 +52,8 @@ class Punctuations(Metric):
 
     name = "punctuations"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["nlp", "text_data", "visualization", "frequency_analysis"],
-    }
+    tasks = ["text_classification", "text_summarization"]
+    tags = ["nlp", "text_data", "visualization", "frequency_analysis"]
 
     def run(self):
         # Can only run this test if we have a Dataset object

validmind/tests/data_validation/nlp/StopWords.py

@@ -71,10 +71,8 @@ class StopWords(ThresholdTest):
     name = "stop_words"
     required_inputs = ["dataset"]
     default_params = {"min_percent_threshold": 0.5, "num_words": 25}
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["nlp", "text_data", "visualization", "frequency_analysis"],
-    }
+    tasks = ["text_classification", "text_summarization"]
+    tags = ["nlp", "text_data", "visualization", "frequency_analysis"]
 
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         # Create a DataFrame from the data

validmind/tests/data_validation/nlp/TextDescription.py

@@ -79,10 +79,8 @@ class TextDescription(Metric):
         "num_top_words": 3,
         "lang": "english",
     }
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["nlp", "text_data", "visualization"],
-    }
+    tasks = ["text_classification", "text_summarization"]
+    tags = ["nlp", "text_data", "visualization"]
 
     def general_text_metrics(self, df, text_column):
         nltk.download("punkt", quiet=True)

validmind/tests/decorator.py

@@ -31,6 +31,8 @@ from validmind.vm_models.figure import (
 )
 from validmind.vm_models.test.result_wrapper import MetricResultWrapper
 
+from ._store import test_store
+
 logger = get_logger(__name__)
 
 
@@ -56,7 +58,9 @@ def _inspect_signature(test_func: callable):
     return inputs, params
 
 
-def _build_result(results, test_id, description, output_template, inputs):  # noqa: C901
+def _build_result(  # noqa: C901
+    results, test_id, description, output_template, inputs, generate_description=True
+):
     ref_id = str(uuid4())
     figure_metadata = {
         "_type": "metric",
@@ -131,6 +135,7 @@ def _build_result(results, test_id, description, output_template, inputs):  # no
                 default_description=description,
                 summary=result_summary.serialize(),
                 figures=figures,
+                should_generate=generate_description,
             )
         ],
         inputs=inputs,
@@ -159,6 +164,7 @@ def _get_run_method(func, inputs, params):
             description=inspect.getdoc(self),
             output_template=self.output_template,
             inputs=self.get_accessed_inputs(),
+            generate_description=self.generate_description,
         )
 
         return self.result
@@ -265,8 +271,6 @@ def test(func_or_id):
         The decorated function.
     """
 
-    from . import _register_custom_test
-
     def decorator(func):
         test_id = func_or_id or f"validmind.custom_metrics.{func.__name__}"
 
@@ -283,13 +287,11 @@ def test(func_or_id):
                 "required_inputs": list(inputs.keys()),
                 "default_params": {k: v["default"] for k, v in params.items()},
                 "__doc__": description,
-                "metadata": {
-                    "task_types": tasks,
-                    "tags": tags,
-                },
+                "tasks": tasks,
+                "tags": tags,
             },
         )
-        _register_custom_test(test_id, metric_class)
+        test_store.register_custom_test(test_id, metric_class)
 
         # special function to allow the function to be saved to a file
         func.save = _get_save_func(func, test_id)

validmind/tests/load.py

@@ -0,0 +1,264 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Module for listing and loading tests."""
+
+import importlib
+import inspect
+import json
+import sys
+from pathlib import Path
+from pprint import pformat
+from uuid import uuid4
+
+import pandas as pd
+from ipywidgets import HTML, Accordion
+
+from ..errors import LoadTestError
+from ..html_templates.content_blocks import test_content_block_html
+from ..logging import get_logger
+from ..unit_metrics.composite import load_composite_metric
+from ..utils import (
+    NumpyEncoder,
+    display,
+    format_dataframe,
+    fuzzy_match,
+    md_to_html,
+    test_id_to_name,
+)
+from .__types__ import TestID
+from ._store import test_provider_store, test_store
+from .decorator import test as test_decorator
+from .utils import test_description
+
+logger = get_logger(__name__)
+
+
+def __init__():
+    directories = [p.name for p in Path(__file__).parent.iterdir() if p.is_dir()]
+
+    for d in directories:
+        for path in Path(__file__).parent.joinpath(d).glob("**/**/*.py"):
+            if path.name.startswith("__") or not path.name[0].isupper():
+                continue  # skip __init__.py and other special files as well as non Test files
+            test_id = (
+                f"validmind.{d}.{path.parent.stem}.{path.stem}"
+                if path.parent.parent.stem == d
+                else f"validmind.{d}.{path.stem}"
+            )
+            test_store.register_test(test_id)
+
+
+__init__()
+
+
+def _pretty_list_tests(tests, truncate=True):
+    table = [
+        {
+            "ID": test_id,
+            "Name": test_id_to_name(test_id),
+            "Description": test_description(test, truncate),
+            "Required Inputs": test.required_inputs,
+            "Params": test.default_params or {},
+        }
+        for test_id, test in tests.items()
+    ]
+
+    return format_dataframe(pd.DataFrame(table))
+
+
+def list_tests(
+    filter=None, task=None, tags=None, pretty=True, truncate=True, __as_class=False
+):
+    """List all tests in the tests directory.
+
+    Args:
+        filter (str, optional): Find tests where the ID, tasks or tags match the
+            filter string. Defaults to None.
+        task (str, optional): Find tests that match the task. Can be used to
+            narrow down matches from the filter string. Defaults to None.
+        tags (list, optional): Find tests that match list of tags. Can be used to
+            narrow down matches from the filter string. Defaults to None.
+        pretty (bool, optional): If True, returns a pandas DataFrame with a
+            formatted table. Defaults to True.
+        truncate (bool, optional): If True, truncates the test description to the first
+            line. Defaults to True. (only used if pretty=True)
+
+    Returns:
+        list or pandas.DataFrame: A list of all tests or a formatted table.
+    """
+    tests = {
+        test_id: load_test(test_id, reload=True)
+        for test_id in test_store.get_test_ids()
+    }
+
+    # first search by the filter string since it's the most general search
+    if filter is not None:
+        tests = {
+            test_id: test
+            for test_id, test in tests.items()
+            if filter.lower() in test_id.lower()
+            or any(filter.lower() in task.lower() for task in test.tasks)
+            or any(fuzzy_match(tag, filter.lower()) for tag in test.tags)
+        }
+
+    # then filter by task type and tags since they are more specific
+    if task is not None:
+        tests = {test_id: test for test_id, test in tests.items() if task in test.tasks}
+
+    if tags is not None:
+        tests = {
+            test_id: test
+            for test_id, test in tests.items()
+            if all(tag in test.tags for tag in tags)
+        }
+
+    if __as_class:
+        return list(tests.values())
+
+    if not pretty:
+        # only return test ids
+        return list(tests.keys())
+
+    return _pretty_list_tests(tests, truncate=truncate)
+
+
+def _load_validmind_test(test_id, reload=False):
+    parts = test_id.split(":")[0].split(".")
+
+    test_module = ".".join(parts[1:-1])
+    test_class = parts[-1]
+
+    error = None
+    test = None
+
+    try:
+        full_path = f"validmind.tests.{test_module}.{test_class}"
+
+        if reload and full_path in sys.modules:
+            module = importlib.reload(sys.modules[full_path])
+        else:
+            module = importlib.import_module(full_path)
+
+        test = getattr(module, test_class)
+    except ModuleNotFoundError as e:
+        error = f"Unable to load test {test_id}. {e}"
+    except AttributeError:
+        error = f"Unable to load test {test_id}. Test not in module: {test_class}"
+
+    return error, test
+
+
+def load_test(test_id: str, reload=False):
+    """Load a test by test ID
+
+    Test IDs are in the format `namespace.path_to_module.TestClassOrFuncName[:result_id]`.
+    The result ID is optional and is used to distinguish between multiple results from the
+    running the same test.
+
+    Args:
+        test_id (str): The test ID in the format `namespace.path_to_module.TestName[:result_id]`
+        reload (bool, optional): Whether to reload the test module. Defaults to False.
+    """
+    # TODO: we should use a dedicated class for test IDs to handle this consistently
+    test_id, result_id = test_id.split(":", 1) if ":" in test_id else (test_id, None)
+
+    error = None
+    namespace = test_id.split(".", 1)[0]
+
+    # TODO: lets implement an extensible loading system instead of this ugly if/else
+    if test_store.get_custom_test(test_id):
+        test = test_store.get_custom_test(test_id)
+
+    elif test_id.startswith("validmind.composite_metric"):
+        error, test = load_composite_metric(test_id)
+
+    elif namespace == "validmind":
+        error, test = _load_validmind_test(test_id, reload=reload)
+
+    elif test_provider_store.has_test_provider(namespace):
+        provider = test_provider_store.get_test_provider(namespace)
+
+        try:
+            test = provider.load_test(test_id.split(".", 1)[1])
+        except Exception as e:
+            error = (
+                f"Unable to load test {test_id} from test provider: "
+                f"{provider}\n Got Exception: {e}"
+            )
+
+    else:
+        error = f"Unable to load test {test_id}. No test provider found."
+
+    if error:
+        logger.error(error)
+        raise LoadTestError(error)
+
+    if inspect.isfunction(test):
+        # if its a function, we decorate it and then load the class
+        # TODO: simplify this as we move towards all functional metrics
+        # "_" is used here so it doesn't conflict with other test ids
+        test_decorator("_")(test)
+        test = test_store.get_custom_test("_")
+
+    test.test_id = f"{test_id}:{result_id}" if result_id else test_id
+
+    return test
+
+
+def describe_test(test_id: TestID = None, raw: bool = False, show: bool = True):
+    """Get or show details about the test
+
+    This function can be used to see test details including the test name, description,
+    required inputs and default params. It can also be used to get a dictionary of the
+    above information for programmatic use.
+
+    Args:
+        test_id (str, optional): The test ID. Defaults to None.
+        raw (bool, optional): If True, returns a dictionary with the test details.
+            Defaults to False.
+    """
+    test = load_test(test_id, reload=True)
+
+    details = {
+        "ID": test_id,
+        "Name": test_id_to_name(test_id),
+        "Required Inputs": test.required_inputs or [],
+        "Params": test.default_params or {},
+        "Description": inspect.getdoc(test).strip() or "",
+    }
+
+    if raw:
+        return details
+
+    html = test_content_block_html.format(
+        test_id=test_id,
+        uuid=str(uuid4()),
+        title=f'{details["Name"]}',
+        description=md_to_html(details["Description"].strip()),
+        required_inputs=", ".join(details["Required Inputs"] or ["None"]),
+        params_table="\n".join(
+            [
+                f"<tr><td>{param}</td><td>{pformat(value, indent=4)}</td></tr>"
+                for param, value in details["Params"].items()
+            ]
+        ),
+        table_display="table" if details["Params"] else "none",
+        example_inputs=json.dumps(
+            {name: f"my_vm_{name}" for name in (details["Required Inputs"] or [])},
+            indent=4,
+        ),
+        example_params=json.dumps(details["Params"] or {}, indent=4, cls=NumpyEncoder),
+        instructions_display="block" if show else "none",
+    )
+
+    if not show:
+        return html
+
+    display(
+        Accordion(
+            children=[HTML(html)],
+            titles=[f"Test Description: {details['Name']} ('{test_id}')"],
+        )
+    )