validmind 2.3.3__py3-none-any.whl → 2.4.0__py3-none-any.whl

validmind/tests/data_validation/TimeSeriesOutliers.py

@@ -4,11 +4,8 @@
 
 from dataclasses import dataclass
 
-import matplotlib.pyplot as plt
 import pandas as pd
-import seaborn as sns
-from ydata_profiling.config import Settings
-from ydata_profiling.model.typeset import ProfilingTypeSet
+import plotly.graph_objects as go
 
 from validmind.vm_models import (
     Figure,
@@ -65,10 +62,8 @@ class TimeSeriesOutliers(ThresholdTest):
     name = "time_series_outliers"
     required_inputs = ["dataset"]
     default_params = {"zscore_threshold": 3}
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data"]
 
     def summary(self, results, all_passed: bool):
         """
@@ -93,7 +88,8 @@ class TimeSeriesOutliers(ThresholdTest):
         zScores = first_result.values["z-score"]
         dates = first_result.values["Date"]
         passFail = [
-            "Pass" if z < self.params["zscore_threshold"] else "Fail" for z in zScores
+            "Pass" if abs(z) < self.params["zscore_threshold"] else "Fail"
+            for z in zScores
         ]
 
         return ResultSummary(
@@ -116,25 +112,26 @@ class TimeSeriesOutliers(ThresholdTest):
         )
 
     def run(self):
+        # Initialize the test_results list
+        test_results = []
+
         # Check if the index of dataframe is datetime
         is_datetime = pd.api.types.is_datetime64_any_dtype(self.inputs.dataset.df.index)
         if not is_datetime:
             raise ValueError("Dataset must be provided with datetime index")
 
-        # Validate threshold paremeter
+        # Validate threshold parameter
         if "zscore_threshold" not in self.params:
             raise ValueError("zscore_threshold must be provided in params")
         zscore_threshold = self.params["zscore_threshold"]
 
         temp_df = self.inputs.dataset.df.copy()
         # temp_df = temp_df.dropna()
-        typeset = ProfilingTypeSet(Settings())
-        dataset_types = typeset.infer_type(temp_df)
-        test_results = []
-        test_figures = []
-        num_features_columns = [
-            k for k, v in dataset_types.items() if str(v) == "Numeric"
-        ]
+
+        # Infer numeric columns
+        num_features_columns = temp_df.select_dtypes(
+            include=["number"]
+        ).columns.tolist()
 
         outliers_table = self.identify_outliers(
             temp_df[num_features_columns], zscore_threshold
@@ -196,49 +193,39 @@ class TimeSeriesOutliers(ThresholdTest):
             df (pandas.DataFrame): Input data with time series.
             outliers_table (pandas.DataFrame): DataFrame with identified outliers.
         Returns:
-            matplotlib.figure.Figure: A matplotlib figure object with subplots for each variable.
+            list: A list of Figure objects with subplots for each variable.
         """
-        sns.set(style="darkgrid")
-        columns = list(self.inputs.dataset.df.columns)
         figures = []
 
-        for col in columns:
-            plt.figure()
-            fig, _ = plt.subplots()
-            column_index_name = df.index.name
-            ax = sns.lineplot(data=df.reset_index(), x=column_index_name, y=col)
+        for col in df.columns:
+            fig = go.Figure()
+
+            fig.add_trace(go.Scatter(x=df.index, y=df[col], mode="lines", name=col))
 
             if not outliers_table.empty:
                 variable_outliers = outliers_table[outliers_table["Variable"] == col]
-                for idx, row in variable_outliers.iterrows():
-                    date = row["Date"]
-                    outlier_value = df.loc[date, col]
-                    ax.scatter(
-                        date,
-                        outlier_value,
-                        marker="o",
-                        s=100,
-                        c="red",
-                        label="Outlier" if idx == 0 else "",
+                fig.add_trace(
+                    go.Scatter(
+                        x=variable_outliers["Date"],
+                        y=df.loc[variable_outliers["Date"], col],
+                        mode="markers",
+                        marker=dict(color="red", size=10),
+                        name="Outlier",
                     )
+                )
 
-            plt.xticks(fontsize=18)
-            plt.yticks(fontsize=18)
-            ax.set_xlabel("")
-            ax.set_ylabel("")
-            ax.set_title(
-                f"Time Series with Outliers for {col}", weight="bold", fontsize=20
+            fig.update_layout(
+                title=f"Time Series with Outliers for {col}",
+                xaxis_title="Date",
+                yaxis_title=col,
             )
 
-            ax.legend()
             figures.append(
                 Figure(
                     for_object=self,
-                    key=f"{self.name}:{col}",
+                    key=f"{self.name}:{col}_{self.inputs.dataset.input_id}",
                     figure=fig,
                 )
             )
 
-        # Do this if you want to prevent the figure from being displayed
-        plt.close("all")
         return figures

validmind/tests/data_validation/TooManyZeroValues.py

@@ -70,10 +70,8 @@ class TooManyZeroValues(ThresholdTest):
     required_inputs = ["dataset"]
     default_params = {"max_percent_threshold": 0.03}
 
-    metadata = {
-        "task_types": ["regression", "classification"],
-        "tags": ["tabular_data"],
-    }
+    tasks = ["regression", "classification"]
+    tags = ["tabular_data"]
 
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         """

validmind/tests/data_validation/UniqueRows.py

@@ -57,10 +57,8 @@ class UniqueRows(ThresholdTest):
     required_inputs = ["dataset"]
     default_params = {"min_percent_threshold": 1}
 
-    metadata = {
-        "task_types": ["regression", "classification"],
-        "tags": ["tabular_data"],
-    }
+    tasks = ["regression", "classification"]
+    tags = ["tabular_data"]
 
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         """

validmind/tests/data_validation/WOEBinPlots.py

@@ -60,10 +60,8 @@ class WOEBinPlots(Metric):
     name = "woe_bin_plots"
     required_inputs = ["dataset"]
     default_params = {"breaks_adj": None, "fig_height": 600, "fig_width": 500}
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["tabular_data", "visualization", "categorical_data"],
-    }
+    tasks = ["classification"]
+    tags = ["tabular_data", "visualization", "categorical_data"]
 
     def run(self):
         df = self.inputs.dataset.df

validmind/tests/data_validation/WOEBinTable.py

@@ -48,10 +48,8 @@ class WOEBinTable(Metric):
     name = "woe_bin_table"
     required_inputs = ["dataset"]
     default_params = {"breaks_adj": None}
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["tabular_data", "categorical_data"],
-    }
+    tasks = ["classification"]
+    tags = ["tabular_data", "categorical_data"]
 
     def run(self):
         target_column = self.inputs.dataset.target_column

validmind/tests/data_validation/ZivotAndrewsArch.py

@@ -51,10 +51,8 @@ class ZivotAndrewsArch(Metric):
 
     name = "zivot_andrews"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data", "stationarity", "unit_root_test"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data", "stationarity", "unit_root_test"]
 
     def run(self):
         """

validmind/tests/data_validation/nlp/CommonWords.py

@@ -53,10 +53,8 @@ class CommonWords(Metric):
 
     name = "common_words"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["nlp", "text_data", "visualization", "frequency_analysis"],
-    }
+    tasks = ["text_classification", "text_summarization"]
+    tags = ["nlp", "text_data", "visualization", "frequency_analysis"]
 
     def run(self):
         # Can only run this test if we have a Dataset object

validmind/tests/data_validation/nlp/Hashtags.py

@@ -56,10 +56,8 @@ class Hashtags(ThresholdTest):
     name = "hashtags"
     required_inputs = ["dataset"]
     default_params = {"top_hashtags": 25}
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["nlp", "text_data", "visualization", "frequency_analysis"],
-    }
+    tasks = ["text_classification", "text_summarization"]
+    tags = ["nlp", "text_data", "visualization", "frequency_analysis"]
 
     def run(self):
         # Can only run this test if we have a Dataset object

validmind/tests/data_validation/nlp/Mentions.py

@@ -56,10 +56,8 @@ class Mentions(ThresholdTest):
 
     required_inputs = ["dataset"]
     default_params = {"top_mentions": 25}
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["nlp", "text_data", "visualization", "frequency_analysis"],
-    }
+    tasks = ["text_classification", "text_summarization"]
+    tags = ["nlp", "text_data", "visualization", "frequency_analysis"]
 
     def run(self):
         # Can only run this test if we have a Dataset object

validmind/tests/data_validation/nlp/Punctuations.py

@@ -52,10 +52,8 @@ class Punctuations(Metric):
 
     name = "punctuations"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["nlp", "text_data", "visualization", "frequency_analysis"],
-    }
+    tasks = ["text_classification", "text_summarization"]
+    tags = ["nlp", "text_data", "visualization", "frequency_analysis"]
 
     def run(self):
         # Can only run this test if we have a Dataset object

validmind/tests/data_validation/nlp/StopWords.py

@@ -71,10 +71,8 @@ class StopWords(ThresholdTest):
     name = "stop_words"
     required_inputs = ["dataset"]
     default_params = {"min_percent_threshold": 0.5, "num_words": 25}
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["nlp", "text_data", "visualization", "frequency_analysis"],
-    }
+    tasks = ["text_classification", "text_summarization"]
+    tags = ["nlp", "text_data", "visualization", "frequency_analysis"]
 
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         # Create a DataFrame from the data

validmind/tests/data_validation/nlp/TextDescription.py

@@ -79,10 +79,8 @@ class TextDescription(Metric):
         "num_top_words": 3,
         "lang": "english",
     }
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["nlp", "text_data", "visualization"],
-    }
+    tasks = ["text_classification", "text_summarization"]
+    tags = ["nlp", "text_data", "visualization"]
 
     def general_text_metrics(self, df, text_column):
         nltk.download("punkt", quiet=True)

validmind/tests/decorator.py

@@ -31,6 +31,8 @@ from validmind.vm_models.figure import (
 )
 from validmind.vm_models.test.result_wrapper import MetricResultWrapper
 
+from ._store import test_store
+
 logger = get_logger(__name__)
 
 
@@ -56,7 +58,9 @@ def _inspect_signature(test_func: callable):
     return inputs, params
 
 
-def _build_result(results, test_id, description, output_template, inputs):  # noqa: C901
+def _build_result(  # noqa: C901
+    results, test_id, description, output_template, inputs, generate_description=True
+):
     ref_id = str(uuid4())
     figure_metadata = {
         "_type": "metric",
@@ -131,6 +135,7 @@ def _build_result(results, test_id, description, output_template, inputs):  # no
                 default_description=description,
                 summary=result_summary.serialize(),
                 figures=figures,
+                should_generate=generate_description,
             )
         ],
         inputs=inputs,
@@ -159,6 +164,7 @@ def _get_run_method(func, inputs, params):
             description=inspect.getdoc(self),
             output_template=self.output_template,
             inputs=self.get_accessed_inputs(),
+            generate_description=self.generate_description,
         )
 
         return self.result
@@ -265,8 +271,6 @@ def test(func_or_id):
         The decorated function.
     """
 
-    from . import _register_custom_test
-
     def decorator(func):
         test_id = func_or_id or f"validmind.custom_metrics.{func.__name__}"
 
@@ -283,13 +287,11 @@ def test(func_or_id):
                 "required_inputs": list(inputs.keys()),
                 "default_params": {k: v["default"] for k, v in params.items()},
                 "__doc__": description,
-                "metadata": {
-                    "task_types": tasks,
-                    "tags": tags,
-                },
+                "tasks": tasks,
+                "tags": tags,
             },
         )
-        _register_custom_test(test_id, metric_class)
+        test_store.register_custom_test(test_id, metric_class)
 
         # special function to allow the function to be saved to a file
         func.save = _get_save_func(func, test_id)

validmind/tests/load.py

@@ -0,0 +1,264 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Module for listing and loading tests."""
+
+import importlib
+import inspect
+import json
+import sys
+from pathlib import Path
+from pprint import pformat
+from uuid import uuid4
+
+import pandas as pd
+from ipywidgets import HTML, Accordion
+
+from ..errors import LoadTestError
+from ..html_templates.content_blocks import test_content_block_html
+from ..logging import get_logger
+from ..unit_metrics.composite import load_composite_metric
+from ..utils import (
+    NumpyEncoder,
+    display,
+    format_dataframe,
+    fuzzy_match,
+    md_to_html,
+    test_id_to_name,
+)
+from .__types__ import TestID
+from ._store import test_provider_store, test_store
+from .decorator import test as test_decorator
+from .utils import test_description
+
+logger = get_logger(__name__)
+
+
+def __init__():
+    directories = [p.name for p in Path(__file__).parent.iterdir() if p.is_dir()]
+
+    for d in directories:
+        for path in Path(__file__).parent.joinpath(d).glob("**/**/*.py"):
+            if path.name.startswith("__") or not path.name[0].isupper():
+                continue  # skip __init__.py and other special files as well as non Test files
+            test_id = (
+                f"validmind.{d}.{path.parent.stem}.{path.stem}"
+                if path.parent.parent.stem == d
+                else f"validmind.{d}.{path.stem}"
+            )
+            test_store.register_test(test_id)
+
+
+__init__()
+
+
+def _pretty_list_tests(tests, truncate=True):
+    table = [
+        {
+            "ID": test_id,
+            "Name": test_id_to_name(test_id),
+            "Description": test_description(test, truncate),
+            "Required Inputs": test.required_inputs,
+            "Params": test.default_params or {},
+        }
+        for test_id, test in tests.items()
+    ]
+
+    return format_dataframe(pd.DataFrame(table))
+
+
+def list_tests(
+    filter=None, task=None, tags=None, pretty=True, truncate=True, __as_class=False
+):
+    """List all tests in the tests directory.
+
+    Args:
+        filter (str, optional): Find tests where the ID, tasks or tags match the
+            filter string. Defaults to None.
+        task (str, optional): Find tests that match the task. Can be used to
+            narrow down matches from the filter string. Defaults to None.
+        tags (list, optional): Find tests that match list of tags. Can be used to
+            narrow down matches from the filter string. Defaults to None.
+        pretty (bool, optional): If True, returns a pandas DataFrame with a
+            formatted table. Defaults to True.
+        truncate (bool, optional): If True, truncates the test description to the first
+            line. Defaults to True. (only used if pretty=True)
+
+    Returns:
+        list or pandas.DataFrame: A list of all tests or a formatted table.
+    """
+    tests = {
+        test_id: load_test(test_id, reload=True)
+        for test_id in test_store.get_test_ids()
+    }
+
+    # first search by the filter string since it's the most general search
+    if filter is not None:
+        tests = {
+            test_id: test
+            for test_id, test in tests.items()
+            if filter.lower() in test_id.lower()
+            or any(filter.lower() in task.lower() for task in test.tasks)
+            or any(fuzzy_match(tag, filter.lower()) for tag in test.tags)
+        }
+
+    # then filter by task type and tags since they are more specific
+    if task is not None:
+        tests = {test_id: test for test_id, test in tests.items() if task in test.tasks}
+
+    if tags is not None:
+        tests = {
+            test_id: test
+            for test_id, test in tests.items()
+            if all(tag in test.tags for tag in tags)
+        }
+
+    if __as_class:
+        return list(tests.values())
+
+    if not pretty:
+        # only return test ids
+        return list(tests.keys())
+
+    return _pretty_list_tests(tests, truncate=truncate)
+
+
+def _load_validmind_test(test_id, reload=False):
+    parts = test_id.split(":")[0].split(".")
+
+    test_module = ".".join(parts[1:-1])
+    test_class = parts[-1]
+
+    error = None
+    test = None
+
+    try:
+        full_path = f"validmind.tests.{test_module}.{test_class}"
+
+        if reload and full_path in sys.modules:
+            module = importlib.reload(sys.modules[full_path])
+        else:
+            module = importlib.import_module(full_path)
+
+        test = getattr(module, test_class)
+    except ModuleNotFoundError as e:
+        error = f"Unable to load test {test_id}. {e}"
+    except AttributeError:
+        error = f"Unable to load test {test_id}. Test not in module: {test_class}"
+
+    return error, test
+
+
+def load_test(test_id: str, reload=False):
+    """Load a test by test ID
+
+    Test IDs are in the format `namespace.path_to_module.TestClassOrFuncName[:result_id]`.
+    The result ID is optional and is used to distinguish between multiple results from the
+    running the same test.
+
+    Args:
+        test_id (str): The test ID in the format `namespace.path_to_module.TestName[:result_id]`
+        reload (bool, optional): Whether to reload the test module. Defaults to False.
+    """
+    # TODO: we should use a dedicated class for test IDs to handle this consistently
+    test_id, result_id = test_id.split(":", 1) if ":" in test_id else (test_id, None)
+
+    error = None
+    namespace = test_id.split(".", 1)[0]
+
+    # TODO: lets implement an extensible loading system instead of this ugly if/else
+    if test_store.get_custom_test(test_id):
+        test = test_store.get_custom_test(test_id)
+
+    elif test_id.startswith("validmind.composite_metric"):
+        error, test = load_composite_metric(test_id)
+
+    elif namespace == "validmind":
+        error, test = _load_validmind_test(test_id, reload=reload)
+
+    elif test_provider_store.has_test_provider(namespace):
+        provider = test_provider_store.get_test_provider(namespace)
+
+        try:
+            test = provider.load_test(test_id.split(".", 1)[1])
+        except Exception as e:
+            error = (
+                f"Unable to load test {test_id} from test provider: "
+                f"{provider}\n Got Exception: {e}"
+            )
+
+    else:
+        error = f"Unable to load test {test_id}. No test provider found."
+
+    if error:
+        logger.error(error)
+        raise LoadTestError(error)
+
+    if inspect.isfunction(test):
+        # if its a function, we decorate it and then load the class
+        # TODO: simplify this as we move towards all functional metrics
+        # "_" is used here so it doesn't conflict with other test ids
+        test_decorator("_")(test)
+        test = test_store.get_custom_test("_")
+
+    test.test_id = f"{test_id}:{result_id}" if result_id else test_id
+
+    return test
+
+
+def describe_test(test_id: TestID = None, raw: bool = False, show: bool = True):
+    """Get or show details about the test
+
+    This function can be used to see test details including the test name, description,
+    required inputs and default params. It can also be used to get a dictionary of the
+    above information for programmatic use.
+
+    Args:
+        test_id (str, optional): The test ID. Defaults to None.
+        raw (bool, optional): If True, returns a dictionary with the test details.
+            Defaults to False.
+    """
+    test = load_test(test_id, reload=True)
+
+    details = {
+        "ID": test_id,
+        "Name": test_id_to_name(test_id),
+        "Required Inputs": test.required_inputs or [],
+        "Params": test.default_params or {},
+        "Description": inspect.getdoc(test).strip() or "",
+    }
+
+    if raw:
+        return details
+
+    html = test_content_block_html.format(
+        test_id=test_id,
+        uuid=str(uuid4()),
+        title=f'{details["Name"]}',
+        description=md_to_html(details["Description"].strip()),
+        required_inputs=", ".join(details["Required Inputs"] or ["None"]),
+        params_table="\n".join(
+            [
+                f"<tr><td>{param}</td><td>{pformat(value, indent=4)}</td></tr>"
+                for param, value in details["Params"].items()
+            ]
+        ),
+        table_display="table" if details["Params"] else "none",
+        example_inputs=json.dumps(
+            {name: f"my_vm_{name}" for name in (details["Required Inputs"] or [])},
+            indent=4,
+        ),
+        example_params=json.dumps(details["Params"] or {}, indent=4, cls=NumpyEncoder),
+        instructions_display="block" if show else "none",
+    )
+
+    if not show:
+        return html
+
+    display(
+        Accordion(
+            children=[HTML(html)],
+            titles=[f"Test Description: {details['Name']} ('{test_id}')"],
+        )
+    )

validmind/tests/metadata.py

@@ -0,0 +1,59 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+
+from validmind.utils import format_dataframe
+
+from .load import list_tests
+
+
+def list_tags():
+    """
+    List unique tags from all test classes.
+    """
+
+    unique_tags = set()
+
+    for test in list_tests(__as_class=True):
+        unique_tags.update(test.tags)
+
+    return list(unique_tags)
+
+
+def list_tasks_and_tags():
+    """
+    List all task types and their associated tags, with one row per task type and
+    all tags for a task type in one row.
+
+    Returns:
+        pandas.DataFrame: A DataFrame with 'Task Type' and concatenated 'Tags'.
+    """
+    task_tags_dict = {}
+
+    for test in list_tests(__as_class=True):
+        for task in test.tasks:
+            task_tags_dict.setdefault(task, set()).update(test.tags)
+
+    return format_dataframe(
+        pd.DataFrame(
+            [
+                {"Task": task, "Tags": ", ".join(tags)}
+                for task, tags in task_tags_dict.items()
+            ]
+        )
+    )
+
+
+def list_tasks():
+    """
+    List unique tasks from all test classes.
+    """
+
+    unique_tasks = set()
+
+    for test in list_tests(__as_class=True):
+        unique_tasks.update(test.tasks)
+
+    return list(unique_tasks)