validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/tests/load.py

@@ -4,12 +4,10 @@
 
 """Module for listing and loading tests."""
 
-import importlib
 import inspect
 import json
-import sys
-from pathlib import Path
 from pprint import pformat
+from typing import List
 from uuid import uuid4
 
 import pandas as pd
@@ -18,39 +16,148 @@ from ipywidgets import HTML, Accordion
 from ..errors import LoadTestError, MissingDependencyError
 from ..html_templates.content_blocks import test_content_block_html
 from ..logging import get_logger
-from ..unit_metrics.composite import load_composite_metric
-from ..utils import (
-    NumpyEncoder,
-    display,
-    format_dataframe,
-    fuzzy_match,
-    md_to_html,
-    test_id_to_name,
-)
+from ..utils import display, format_dataframe, fuzzy_match, md_to_html, test_id_to_name
+from ..vm_models import VMDataset, VMModel
 from .__types__ import TestID
 from ._store import test_provider_store, test_store
-from .decorator import test as test_decorator
-from .utils import test_description
 
 logger = get_logger(__name__)
 
 
-def __init__():
-    directories = [p.name for p in Path(__file__).parent.iterdir() if p.is_dir()]
+INPUT_TYPE_MAP = {
+    "dataset": VMDataset,
+    "datasets": List[VMDataset],
+    "model": VMModel,
+    "models": List[VMModel],
+}
 
-    for d in directories:
-        for path in Path(__file__).parent.joinpath(d).glob("**/**/*.py"):
-            if path.name.startswith("__") or not path.name[0].isupper():
-                continue  # skip __init__.py and other special files as well as non Test files
-            test_id = (
-                f"validmind.{d}.{path.parent.stem}.{path.stem}"
-                if path.parent.parent.stem == d
-                else f"validmind.{d}.{path.stem}"
-            )
-            test_store.register_test(test_id)
 
+def _inspect_signature(test_func: callable):
+    inputs = {}
+    params = {}
 
-__init__()
+    for name, arg in inspect.signature(test_func).parameters.items():
+        if name in INPUT_TYPE_MAP:
+            inputs[name] = {"type": INPUT_TYPE_MAP[name]}
+        elif name == "args" or name == "kwargs":
+            continue
+        else:
+            params[name] = {
+                "type": (
+                    arg.annotation.__name__
+                    if arg.annotation and hasattr(arg.annotation, "__name__")
+                    else None
+                ),
+                "default": (
+                    arg.default if arg.default is not inspect.Parameter.empty else None
+                ),
+            }
+
+    return inputs, params
+
+
+def load_test(test_id: str, test_func: callable = None, reload: bool = False):
+    """Load a test by test ID
+
+    Test IDs are in the format `namespace.path_to_module.TestClassOrFuncName[:tag]`.
+    The tag is optional and is used to distinguish between multiple results from the
+    same test.
+
+    Args:
+        test_id (str): The test ID in the format `namespace.path_to_module.TestName[:tag]`
+        test_func (callable, optional): The test function to load. If not provided, the
+            test will be loaded from the test provider. Defaults to None.
+    """
+    # remove tag if present
+    test_id = test_id.split(":", 1)[0]
+    namespace = test_id.split(".", 1)[0]
+
+    # if not already loaded, load it from appropriate provider
+    if test_id not in test_store.tests or reload:
+        if test_id.startswith("validmind.composite_metric"):
+            # TODO: add composite metric loading
+            pass
+
+        if not test_func:
+            if not test_provider_store.has_test_provider(namespace):
+                raise LoadTestError(
+                    f"No test provider found for namespace: {namespace}"
+                )
+
+            provider = test_provider_store.get_test_provider(namespace)
+
+            try:
+                test_func = provider.load_test(test_id.split(".", 1)[1])
+            except Exception as e:
+                raise LoadTestError(
+                    f"Unable to load test '{test_id}' from {namespace} test provider",
+                    original_error=e,
+                ) from e
+
+        # add test_id as an attribute to the test function
+        test_func.test_id = test_id
+
+        # fallback to using func name if no docstring is found
+        if not inspect.getdoc(test_func):
+            test_func.__doc__ = f"{test_func.__name__} ({test_id})"
+
+        # add inputs and params as attributes to the test function
+        test_func.inputs, test_func.params = _inspect_signature(test_func)
+
+        test_store.register_test(test_id, test_func)
+
+    return test_store.get_test(test_id)
+
+
+def _list_test_ids():
+    test_ids = []
+
+    for namespace, test_provider in test_provider_store.test_providers.items():
+        test_ids.extend(
+            [f"{namespace}.{test_id}" for test_id in sorted(test_provider.list_tests())]
+        )
+
+    return test_ids
+
+
+def _load_tests(test_ids):
+    """Load a set of tests, handling missing dependencies."""
+    tests = {}
+
+    for test_id in test_ids:
+        try:
+            tests[test_id] = load_test(test_id)
+        except LoadTestError as e:
+            if not e.original_error or not isinstance(
+                e.original_error, MissingDependencyError
+            ):
+                raise e
+
+            e = e.original_error
+
+            logger.debug(str(e))
+
+            if e.extra:
+                logger.info(
+                    f"Skipping `{test_id}` as it requires extra dependencies: {e.required_dependencies}."
+                    f" Please run `pip install validmind[{e.extra}]` to view and run this test."
+                )
+            else:
+                logger.info(
+                    f"Skipping `{test_id}` as it requires missing dependencies: {e.required_dependencies}."
+                    " Please install the missing dependencies to view and run this test."
+                )
+
+    return tests
+
+
+def _test_description(test_description: str, num_lines: int = 5):
+    description = test_description.strip("\n").strip()
+
+    if len(description.split("\n")) > num_lines:
+        return description.strip().split("\n")[0] + "..."
+
+    return description
 
 
 def _pretty_list_tests(tests, truncate=True):
@@ -58,9 +165,12 @@ def _pretty_list_tests(tests, truncate=True):
         {
             "ID": test_id,
             "Name": test_id_to_name(test_id),
-            "Description": test_description(test, truncate),
-            "Required Inputs": test.required_inputs,
-            "Params": test.default_params or {},
+            "Description": _test_description(
+                inspect.getdoc(test),
+                num_lines=(5 if truncate else 999999),
+            ),
+            "Required Inputs": test.inputs,
+            "Params": test.params,
         }
         for test_id, test in tests.items()
     ]
@@ -68,9 +178,57 @@ def _pretty_list_tests(tests, truncate=True):
     return format_dataframe(pd.DataFrame(table))
 
 
-def list_tests(
-    filter=None, task=None, tags=None, pretty=True, truncate=True, __as_class=False
-):
+def list_tags():
+    """
+    List unique tags from all test classes.
+    """
+
+    unique_tags = set()
+
+    for test in _load_tests(list_tests(pretty=False)):
+        unique_tags.update(test.__tags__)
+
+    return list(unique_tags)
+
+
+def list_tasks_and_tags():
+    """
+    List all task types and their associated tags, with one row per task type and
+    all tags for a task type in one row.
+
+    Returns:
+        pandas.DataFrame: A DataFrame with 'Task Type' and concatenated 'Tags'.
+    """
+    task_tags_dict = {}
+
+    for test in _load_tests(list_tests(pretty=False)):
+        for task in test.__tasks__:
+            task_tags_dict.setdefault(task, set()).update(test.__tags__)
+
+    return format_dataframe(
+        pd.DataFrame(
+            [
+                {"Task": task, "Tags": ", ".join(tags)}
+                for task, tags in task_tags_dict.items()
+            ]
+        )
+    )
+
+
+def list_tasks():
+    """
+    List unique tasks from all test classes.
+    """
+
+    unique_tasks = set()
+
+    for test in _load_tests(list_tests(pretty=False)):
+        unique_tasks.update(test.__tasks__)
+
+    return list(unique_tasks)
+
+
+def list_tests(filter=None, task=None, tags=None, pretty=True, truncate=True):
     """List all tests in the tests directory.
 
     Args:
@@ -88,30 +246,13 @@ def list_tests(
     Returns:
         list or pandas.DataFrame: A list of all tests or a formatted table.
     """
-    # tests = {
-    #     test_id: load_test(test_id, reload=True)
-    #     for test_id in test_store.get_test_ids()
-    # }
-    tests = {}
-    for test_id in test_store.get_test_ids():
-        try:
-            tests[test_id] = load_test(test_id, reload=True)
-        except MissingDependencyError as e:
-            # skip tests that have missing dependencies
-            logger.debug(str(e))
+    test_ids = _list_test_ids()
 
-            if e.extra:
-                logger.info(
-                    f"Skipping `{test_id}` as it requires extra dependencies: {e.required_dependencies}."
-                    f" Please run `pip install validmind[{e.extra}]` to view and run this test."
-                )
-            else:
-                logger.info(
-                    f"Skipping `{test_id}` as it requires missing dependencies: {e.required_dependencies}."
-                    " Please install the missing dependencies to view and run this test."
-                )
+    # no need to load test funcs (takes a while) if we're just returning the test ids
+    if not filter and not task and not tags and not pretty:
+        return test_ids
 
-            continue
+    tests = _load_tests(test_ids)
 
     # first search by the filter string since it's the most general search
     if filter is not None:
@@ -119,114 +260,29 @@ def list_tests(
             test_id: test
             for test_id, test in tests.items()
             if filter.lower() in test_id.lower()
-            or any(filter.lower() in task.lower() for task in test.tasks)
-            or any(fuzzy_match(tag, filter.lower()) for tag in test.tags)
+            or any(filter.lower() in task.lower() for task in test.__tasks__)
+            or any(fuzzy_match(tag, filter.lower()) for tag in test.__tags__)
         }
 
     # then filter by task type and tags since they are more specific
     if task is not None:
-        tests = {test_id: test for test_id, test in tests.items() if task in test.tasks}
+        tests = {
+            test_id: test for test_id, test in tests.items() if task in test.__tasks__
+        }
 
     if tags is not None:
         tests = {
             test_id: test
             for test_id, test in tests.items()
-            if all(tag in test.tags for tag in tags)
+            if all(tag in test.__tags__ for tag in tags)
         }
 
-    if __as_class:
-        return list(tests.values())
-
     if not pretty:
-        # only return test ids
         return list(tests.keys())
 
     return _pretty_list_tests(tests, truncate=truncate)
 
 
-def _load_validmind_test(test_id, reload=False):
-    parts = test_id.split(":")[0].split(".")
-
-    test_module = ".".join(parts[1:-1])
-    test_class = parts[-1]
-
-    error = None
-    test = None
-
-    try:
-        full_path = f"validmind.tests.{test_module}.{test_class}"
-
-        if reload and full_path in sys.modules:
-            module = importlib.reload(sys.modules[full_path])
-        else:
-            module = importlib.import_module(full_path)
-
-        test = getattr(module, test_class)
-    except ModuleNotFoundError as e:
-        error = f"Unable to load test {test_id}. {e}"
-    except AttributeError:
-        error = f"Unable to load test {test_id}. Test not in module: {test_class}"
-
-    return error, test
-
-
-def load_test(test_id: str, reload=False):
-    """Load a test by test ID
-
-    Test IDs are in the format `namespace.path_to_module.TestClassOrFuncName[:result_id]`.
-    The result ID is optional and is used to distinguish between multiple results from the
-    running the same test.
-
-    Args:
-        test_id (str): The test ID in the format `namespace.path_to_module.TestName[:result_id]`
-        reload (bool, optional): Whether to reload the test module. Defaults to False.
-    """
-    # TODO: we should use a dedicated class for test IDs to handle this consistently
-    test_id, result_id = test_id.split(":", 1) if ":" in test_id else (test_id, None)
-
-    error = None
-    namespace = test_id.split(".", 1)[0]
-
-    # TODO: lets implement an extensible loading system instead of this ugly if/else
-    if test_store.get_custom_test(test_id):
-        test = test_store.get_custom_test(test_id)
-
-    elif test_id.startswith("validmind.composite_metric"):
-        error, test = load_composite_metric(test_id)
-
-    elif namespace == "validmind":
-        error, test = _load_validmind_test(test_id, reload=reload)
-
-    elif test_provider_store.has_test_provider(namespace):
-        provider = test_provider_store.get_test_provider(namespace)
-
-        try:
-            test = provider.load_test(test_id.split(".", 1)[1])
-        except Exception as e:
-            error = (
-                f"Unable to load test {test_id} from test provider: "
-                f"{provider}\n Got Exception: {e}"
-            )
-
-    else:
-        error = f"Unable to load test {test_id}. No test provider found."
-
-    if error:
-        logger.error(error)
-        raise LoadTestError(error)
-
-    if inspect.isfunction(test):
-        # if its a function, we decorate it and then load the class
-        # TODO: simplify this as we move towards all functional metrics
-        # "_" is used here so it doesn't conflict with other test ids
-        test_decorator("_")(test)
-        test = test_store.get_custom_test("_")
-
-    test.test_id = f"{test_id}:{result_id}" if result_id else test_id
-
-    return test
-
-
 def describe_test(test_id: TestID = None, raw: bool = False, show: bool = True):
     """Get or show details about the test
 
@@ -239,13 +295,13 @@ def describe_test(test_id: TestID = None, raw: bool = False, show: bool = True):
         raw (bool, optional): If True, returns a dictionary with the test details.
             Defaults to False.
     """
-    test = load_test(test_id, reload=True)
+    test = load_test(test_id)
 
     details = {
         "ID": test_id,
         "Name": test_id_to_name(test_id),
-        "Required Inputs": test.required_inputs or [],
-        "Params": test.default_params or {},
+        "Required Inputs": test.inputs or [],
+        "Params": test.params or {},
         "Description": inspect.getdoc(test).strip() or "",
     }
 
@@ -260,8 +316,8 @@ def describe_test(test_id: TestID = None, raw: bool = False, show: bool = True):
         required_inputs=", ".join(details["Required Inputs"] or ["None"]),
         params_table="\n".join(
             [
-                f"<tr><td>{param}</td><td>{pformat(value, indent=4)}</td></tr>"
-                for param, value in details["Params"].items()
+                f"<tr><td>{param}</td><td>{pformat(param_spec['default'], indent=4)}</td></tr>"
+                for param, param_spec in details["Params"].items()
             ]
         ),
         table_display="table" if details["Params"] else "none",
@@ -269,7 +325,10 @@ def describe_test(test_id: TestID = None, raw: bool = False, show: bool = True):
             {name: f"my_vm_{name}" for name in (details["Required Inputs"] or [])},
             indent=4,
         ),
-        example_params=json.dumps(details["Params"] or {}, indent=4, cls=NumpyEncoder),
+        example_params=json.dumps(
+            {param: f"my_vm_{param}" for param in (details["Params"] or {}).keys()},
+            indent=4,
+        ),
         instructions_display="block" if show else "none",
     )
 
@@ -279,6 +338,6 @@ def describe_test(test_id: TestID = None, raw: bool = False, show: bool = True):
     display(
         Accordion(
             children=[HTML(html)],
-            titles=[f"Test Description: {details['Name']} ('{test_id}')"],
+            titles=[f"Test: {details['Name']} ('{test_id}')"],
         )
     )

validmind/tests/model_validation/BertScore.py

@@ -7,11 +7,16 @@ import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import tags, tasks
+from validmind.tests.utils import validate_prediction
 
 
 @tags("nlp", "text_data", "visualization")
 @tasks("text_classification", "text_summarization")
-def BertScore(dataset, model):
+def BertScore(
+    dataset,
+    model,
+    evaluation_model="distilbert-base-uncased",
+):
     """
     Assesses the quality of machine-generated text using BERTScore metrics and visualizes results through histograms
     and bar charts, alongside compiling a comprehensive table of descriptive statistics.
@@ -29,7 +34,10 @@ def BertScore(dataset, model):
     BERTScore metrics and compiles them into a dataframe. Histograms and bar charts are generated for each BERTScore
     metric (Precision, Recall, and F1 Score) to visualize their distribution. Additionally, a table of descriptive
     statistics (mean, median, standard deviation, minimum, and maximum) is compiled for each metric, providing a
-    comprehensive summary of the model's performance.
+    comprehensive summary of the model's performance. The test uses the `evaluation_model` param to specify the
+    huggingface model to use for evaluation. `microsoft/deberta-xlarge-mnli` is the best-performing model but is
+    very large and may be slow without a GPU. `microsoft/deberta-large-mnli` is a smaller model that is faster to
+    run and `distilbert-base-uncased` is much lighter and can run on a CPU but is less accurate.
 
     ### Signs of High Risk
 
@@ -61,11 +69,8 @@ def BertScore(dataset, model):
     y_true = dataset.y
     y_pred = dataset.y_pred(model)
 
-    # Ensure y_true and y_pred have the same length
-    if len(y_true) != len(y_pred):
-        min_length = min(len(y_true), len(y_pred))
-        y_true = y_true[:min_length]
-        y_pred = y_pred[:min_length]
+    # Ensure equal lengths and get truncated data if necessary
+    y_true, y_pred = validate_prediction(y_true, y_pred)
 
     # Load the BERT evaluation metric
     bert = evaluate.load("bertscore")
@@ -75,6 +80,7 @@ def BertScore(dataset, model):
         predictions=y_pred,
         references=y_true,
         lang="en",
+        model_type=evaluation_model,
     )
 
     # Convert scores to a dataframe

validmind/tests/model_validation/BleuScore.py

@@ -7,6 +7,7 @@ import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import tags, tasks
+from validmind.tests.utils import validate_prediction
 
 
 @tags("nlp", "text_data", "visualization")
@@ -61,6 +62,9 @@ def BleuScore(dataset, model):
     y_true = dataset.y
     y_pred = dataset.y_pred(model)
 
+    # Ensure equal lengths and get truncated data if necessary
+    y_true, y_pred = validate_prediction(y_true, y_pred)
+
     # Load the BLEU evaluation metric
     bleu = evaluate.load("bleu")
 

validmind/tests/model_validation/ClusterSizeDistribution.py

@@ -2,16 +2,16 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
-@dataclass
-class ClusterSizeDistribution(Metric):
+@tags("sklearn", "model_performance")
+@tasks("clustering")
+def ClusterSizeDistribution(dataset: VMDataset, model: VMModel):
     """
     Assesses the performance of clustering models by comparing the distribution of cluster sizes in model predictions
     with the actual data.
@@ -52,47 +52,24 @@ class ClusterSizeDistribution(Metric):
     - May not fully capture other important aspects of clustering, such as cluster density, distances between clusters,
     and the shape of clusters.
     """
-
-    name = "cluster_size_distribution"
-    required_inputs = ["model", "dataset"]
-    tasks = ["clustering"]
-    tags = [
-        "sklearn",
-        "model_performance",
-    ]
-
-    def run(self):
-        y_true_train = self.inputs.dataset.y
-        y_pred_train = self.inputs.dataset.y_pred(self.inputs.model)
-        y_true_train = y_true_train.astype(y_pred_train.dtype)
-        df = pd.DataFrame(
-            {"Actual": y_true_train.ravel(), "Prediction": y_pred_train.ravel()}
-        )
-        df_counts = df.apply(pd.value_counts)
-
-        fig = go.Figure(
-            data=[
-                go.Bar(name="Actual", x=df_counts.index, y=df_counts["Actual"].values),
-                go.Bar(
-                    name="Prediction",
-                    x=df_counts.index,
-                    y=df_counts["Prediction"].values,
-                ),
-            ]
-        )
-        # Change the bar mode
-        fig.update_xaxes(title_text="Number of clusters", showgrid=False)
-        fig.update_yaxes(title_text="Counts", showgrid=False)
-        fig.update_layout(
-            title_text="Cluster distribution", title_x=0.5, barmode="group"
-        )
-
-        figures = [
-            Figure(
-                for_object=self,
-                key=self.key,
-                figure=fig,
-            )
+    y_pred = dataset.y_pred(model)
+    y_true = dataset.y.astype(y_pred.dtype)
+
+    df = pd.DataFrame({"Actual": y_true.ravel(), "Prediction": y_pred.ravel()})
+    df_counts = df.apply(pd.value_counts)
+
+    fig = go.Figure(
+        data=[
+            go.Bar(name="Actual", x=df_counts.index, y=df_counts["Actual"].values),
+            go.Bar(
+                name="Prediction",
+                x=df_counts.index,
+                y=df_counts["Prediction"].values,
+            ),
         ]
+    )
+    fig.update_xaxes(title_text="Number of clusters", showgrid=False)
+    fig.update_yaxes(title_text="Counts", showgrid=False)
+    fig.update_layout(title_text="Cluster distribution", title_x=0.5, barmode="group")
 
-        return self.cache_results(figures=figures)
+    return fig

validmind/tests/model_validation/ContextualRecall.py

@@ -7,6 +7,7 @@ import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import tags, tasks
+from validmind.tests.utils import validate_prediction
 
 
 @tags("nlp", "text_data", "visualization")
@@ -64,6 +65,8 @@ def ContextualRecall(dataset, model):
     y_true = dataset.y
     y_pred = dataset.y_pred(model)
 
+    validate_prediction(y_true, y_pred)
+
     score_list = []
     for y_t, y_p in zip(y_true, y_pred):
         # Tokenize the reference and candidate texts