validmind 2.5.25__py3-none-any.whl → 2.6.8__py3-none-any.whl

validmind/tests/test_providers.py

@@ -4,28 +4,46 @@
 
 import importlib.util
 import os
+import re
 import sys
-from typing import Protocol
+from pathlib import Path
+from typing import List, Protocol
 
 from validmind.logging import get_logger
 
-from ._store import test_provider_store
-
 logger = get_logger(__name__)
 
+# list all files in directory of this file
+__private_files = [f.name for f in Path(__file__).parent.glob("*.py")]
+
+
+def _is_test_file(path: Path) -> bool:
+    return (
+        path.name[0].isupper()
+        or re.search(r"def\s*" + re.escape(path.stem), path.read_text())
+    ) and path.name not in __private_files
+
 
 class TestProvider(Protocol):
     """Protocol for user-defined test providers"""
 
-    def load_test(self, test_id: str):
-        """Load the test by test ID
+    def list_tests(self) -> List[str]:
+        """List all tests in the given namespace
+
+        Returns:
+            list: A list of test IDs
+        """
+        ...
+
+    def load_test(self, test_id: str) -> callable:
+        """Load the test function identified by the given test_id
 
         Args:
             test_id (str): The test ID (does not contain the namespace under which
                 the test is registered)
 
         Returns:
-            Test: A test class or function
+            callable: The test function
 
         Raises:
             FileNotFoundError: If the test is not found
@@ -33,22 +51,6 @@ class TestProvider(Protocol):
         ...
 
 
-class LocalTestProviderLoadModuleError(Exception):
-    """
-    When the local file module can't be loaded.
-    """
-
-    pass
-
-
-class LocalTestProviderLoadTestError(Exception):
-    """
-    When local file module was loaded but the test class can't be located.
-    """
-
-    pass
-
-
 class LocalTestProvider:
     """
     Test providers in ValidMind are responsible for loading tests from different sources,
@@ -69,6 +71,11 @@ class LocalTestProvider:
     # Register the test provider with a namespace
     register_test_provider("my_namespace", test_provider)
 
+    # List all tests in the namespace (returns a list of test IDs)
+    test_provider.list_tests()
+    # this is used by the validmind.tests.list_tests() function to aggregate all tests
+    # from all test providers
+
     # Load a test using the test_id (namespace + path to test class module)
     test = test_provider.load_test("my_namespace.my_test_class")
     # full path to the test class module is /path/to/tests/folder/my_test_class.py
@@ -86,7 +93,32 @@ class LocalTestProvider:
         Args:
             root_folder (str): The root directory for local tests.
         """
-        self.root_folder = root_folder
+        self.root_folder = os.path.abspath(root_folder)
+
+    def list_tests(self):
+        """List all tests in the given namespace
+
+        Returns:
+            list: A list of test IDs
+        """
+        test_ids = []
+
+        for root, _, files in os.walk(self.root_folder):
+            for filename in files:
+                if not filename.endswith(".py") or filename.startswith("__"):
+                    continue
+
+                path = Path(root) / filename
+                if not _is_test_file(path):
+                    continue
+
+                rel_path = path.relative_to(self.root_folder)
+
+                test_id_parts = [p.stem for p in rel_path.parents if p.stem][::-1]
+                test_id_parts.append(path.stem)
+                test_ids.append(".".join(test_id_parts))
+
+        return sorted(test_ids)
 
     def load_test(self, test_id: str):
         """
@@ -100,60 +132,58 @@ class LocalTestProvider:
             The test class that matches the last part of the test_id.
 
         Raises:
-            Exception: If the test can't be imported or loaded.
+            LocalTestProviderLoadModuleError: If the test module cannot be imported
+            LocalTestProviderLoadTestError: If the test class cannot be found in the module
         """
-        test_path = f"{test_id.replace('.', '/')}.py"
-        file_path = os.path.join(self.root_folder, test_path)
-
-        logger.debug(f"Loading test {test_id} from {file_path}")
-
-        # Check if the module uses relative imports
-        with open(file_path, "r") as file:
-            lines = file.readlines()
-
-        # handle test with relative imports
-        if any(line.strip().startswith("from .") for line in lines):
-            logger.debug("Found relative imports, using alternative import method")
-
-            parent_folder = os.path.dirname(file_path)
-            if parent_folder not in sys.path:
-                sys.path.append(os.path.dirname(parent_folder))
-
-            try:
-                module = importlib.import_module(
-                    f"{os.path.basename(parent_folder)}.{test_id.split('.')[-1]}"
-                )
-            except Exception as e:
-                # error will be handled/re-raised by `load_test` func
-                raise LocalTestProviderLoadModuleError(
-                    f"Failed to load the module from {file_path}. Error: {str(e)}"
-                )
-
-        else:
-            try:
-                spec = importlib.util.spec_from_file_location(test_id, file_path)
-                module = importlib.util.module_from_spec(spec)
-                spec.loader.exec_module(module)
-            except Exception as e:
-                # error will be handled/re-raised by `load_test` func
-                raise LocalTestProviderLoadModuleError(
-                    f"Failed to load the module from {file_path}. Error: {str(e)}"
-                )
-
-        try:
-            # find the test class that matches the last part of the test_id
-            return getattr(module, test_id.split(".")[-1])
-        except AttributeError as e:
-            raise LocalTestProviderLoadTestError(
-                f"Failed to find the test class in the module. Error: {str(e)}"
-            )
-
-
-def register_test_provider(namespace: str, test_provider: "TestProvider") -> None:
-    """Register an external test provider
-
-    Args:
-        namespace (str): The namespace of the test provider
-        test_provider (TestProvider): The test provider
-    """
-    test_provider_store.register_test_provider(namespace, test_provider)
+        # Convert test_id to file path
+        file_path = os.path.join(self.root_folder, f"{test_id.replace('.', '/')}.py")
+        file_path = os.path.abspath(file_path)
+
+        module_dir = os.path.dirname(file_path)
+        module_name = test_id.split(".")[-1]
+
+        # module specification
+        spec = importlib.util.spec_from_file_location(
+            name=module_name,
+            location=file_path,
+            submodule_search_locations=[module_dir],
+        )
+
+        # module instance from specification
+        module = importlib.util.module_from_spec(spec)
+
+        # add module to sys.modules
+        sys.modules[module_name] = module
+        # execute the module
+        spec.loader.exec_module(module)
+
+        # test function should match the module (file) name exactly
+        return getattr(module, module_name)
+
+
+class ValidMindTestProvider:
+    """Test provider for ValidMind tests"""
+
+    def __init__(self):
+        # two subproviders: unit_metrics and normal tests
+        self.metrics_provider = LocalTestProvider(
+            os.path.join(os.path.dirname(__file__), "..", "unit_metrics")
+        )
+        self.tests_provider = LocalTestProvider(os.path.dirname(__file__))
+
+    def list_tests(self) -> List[str]:
+        """List all tests in the ValidMind test provider"""
+        metric_ids = [
+            f"unit_metrics.{test}" for test in self.metrics_provider.list_tests()
+        ]
+        test_ids = self.tests_provider.list_tests()
+
+        return metric_ids + test_ids
+
+    def load_test(self, test_id: str) -> callable:
+        """Load a ValidMind test or unit metric"""
+        return (
+            self.metrics_provider.load_test(test_id.replace("unit_metrics.", ""))
+            if test_id.startswith("unit_metrics.")
+            else self.tests_provider.load_test(test_id)
+        )

validmind/tests/utils.py

@@ -6,6 +6,13 @@
 
 import inspect
 
+import numpy as np
+import pandas as pd
+
+from validmind.logging import get_logger
+
+logger = get_logger(__name__)
+
 
 def test_description(test_class, truncate=True):
     description = inspect.getdoc(test_class).strip()
@@ -14,3 +21,87 @@ def test_description(test_class, truncate=True):
         return description.strip().split("\n")[0] + "..."
 
     return description
+
+
+def remove_nan_pairs(y_true, y_pred, dataset_id=None):
+    """
+    Remove pairs where either true or predicted values are NaN/None.
+    Args:
+        y_true: List or array of true values
+        y_pred: List or array of predicted values
+        dataset_id: Optional identifier for the dataset (for logging)
+    Returns:
+        tuple: (cleaned_y_true, cleaned_y_pred)
+    """
+    # Convert to numpy arrays for easier handling
+    y_true = np.array(y_true)
+    y_pred = np.array(y_pred)
+
+    # Find indices where either value is NaN/None
+    nan_mask = ~(pd.isnull(y_true) | pd.isnull(y_pred))
+    nan_count = len(y_true) - np.sum(nan_mask)
+
+    if nan_count > 0:
+        dataset_info = f" from dataset '{dataset_id}'" if dataset_id else ""
+        logger.warning(
+            f"Found {nan_count} row(s){dataset_info} with NaN/None values. "
+            f"Removing these pairs. {len(y_true)} -> {np.sum(nan_mask)} pairs remaining."
+        )
+        return y_true[nan_mask], y_pred[nan_mask]
+
+    return y_true, y_pred
+
+
+def ensure_equal_lengths(y_true, y_pred, dataset_id=None):
+    """
+    Check if true and predicted values have matching lengths, log warning if they don't,
+    and truncate to the shorter length if necessary. Also removes any NaN/None values.
+
+    Args:
+        y_true: List or array of true values
+        y_pred: List or array of predicted values
+        dataset_id: Optional identifier for the dataset (for logging)
+
+    Returns:
+        tuple: (cleaned_y_true, cleaned_y_pred)
+    """
+    # First remove any NaN values
+    y_true, y_pred = remove_nan_pairs(y_true, y_pred, dataset_id)
+
+    # Then handle length mismatches
+    if len(y_true) != len(y_pred):
+        dataset_info = f" from dataset '{dataset_id}'" if dataset_id else ""
+        min_length = min(len(y_true), len(y_pred))
+        logger.warning(
+            f"Length mismatch{dataset_info}: "
+            f"true values ({len(y_true)}) != predicted values ({len(y_pred)}). "
+            f"Truncating to first {min_length} pairs."
+        )
+        return y_true[:min_length], y_pred[:min_length]
+
+    return y_true, y_pred
+
+
+def validate_prediction(y_true, y_pred, dataset_id=None):
+    """
+    Comprehensive validation of true and predicted value pairs.
+    Handles NaN/None values and length mismatches.
+
+    Args:
+        y_true: List or array of true values
+        y_pred: List or array of predicted values
+        dataset_id: Optional identifier for the dataset (for logging)
+
+    Returns:
+        tuple: (cleaned_y_true, cleaned_y_pred) with matching lengths and no NaN values
+
+    Example:
+        >>> y_true, y_pred = validate_prediction_pairs(dataset.y, model.predict(dataset.X), dataset.input_id)
+    """
+    # First remove any NaN values
+    y_true, y_pred = remove_nan_pairs(y_true, y_pred, dataset_id)
+
+    # Then handle any length mismatches
+    y_true, y_pred = ensure_equal_lengths(y_true, y_pred, dataset_id)
+
+    return y_true, y_pred

validmind/unit_metrics/__init__.py

@@ -2,170 +2,31 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-import glob
-import hashlib
-import json
-import os
-from importlib import import_module
-from textwrap import dedent
+from validmind.tests._store import test_provider_store
+from validmind.tests.load import describe_test
+from validmind.tests.run import run_test
 
-from IPython.display import Markdown, display
 
-from validmind.input_registry import input_registry
-from validmind.tests.decorator import _build_result, _inspect_signature
-from validmind.utils import test_id_to_name
+def list_metrics(**kwargs):
+    """List all metrics"""
+    vm_provider = test_provider_store.get_test_provider("validmind")
+    vm_metrics_provider = vm_provider.metrics_provider
 
-unit_metric_results_cache = {}
+    prefix = "validmind.unit_metrics."
 
-
-def _serialize_dataset(dataset, model=None, sample_size=1000):
-    columns = [*dataset.feature_columns, dataset.target_column]
-    if model:
-        columns.append(dataset.prediction_column(model))
-
-    df = dataset._df[columns]
-
-    return hashlib.md5(
-        df.sample(n=min(sample_size, df.shape[0]), random_state=42)
-        .to_string(header=True, index=True)
-        .encode()
-    ).hexdigest()
-
-
-def _get_metric_cache_key(metric_id, inputs, params):
-    cache_elements = [
-        metric_id,
-        hashlib.md5(json.dumps(params, sort_keys=True).encode()).hexdigest(),
-    ]
-
-    if "model" in inputs:
-        cache_elements.append(inputs["model"].input_id)
-
-    if "dataset" in inputs:
-        cache_elements.append(inputs["dataset"].input_id)
-        cache_elements.append(
-            _serialize_dataset(inputs["dataset"], inputs.get("model"))
-        )
-
-    return hashlib.md5("_".join(cache_elements).encode()).hexdigest()
-
-
-def describe_metric(metric_id, raw=False):
-    """Describe a metric
-
-    Args:
-        metric_id (str): The metric id (e.g. 'validmind.unit_metrics.classification.F1')
-        raw (bool): Whether to return the description as a dictionary
-
-    Returns:
-        dict: A dictionary containing the metric description
-    """
-    metric = load_metric(metric_id)
-    inputs, params = _inspect_signature(metric)
-
-    if raw:
-        return {
-            "id": metric_id,
-            "description": metric.__doc__,
-            "inputs": inputs,
-            "params": params,
-        }
-
-    inputs = ", ".join(inputs.keys())
-    params = ", ".join(params.keys())
-    description_md = f"""
-    ### {test_id_to_name(metric_id)} (*'{metric_id}'*)
-
-    {metric.__doc__ or ""}
-
-    **Inputs**: {inputs}
-
-    **Parameters**: {params}
-    """
-    display(Markdown(dedent(description_md)))
-
-
-def list_metrics():
-    """List all available metrics
-
-    Returns:
-        list: A list of metric ids
-    """
-    # current directory of this file is the __init__.py file in the validmind/unit_metrics directory
-    # glob for all metrics in the unit_metrics directory (indicated by capitalized python files)
-    # recursive since we want to include subdirectories
-    curr_dir = os.path.dirname(os.path.realpath(__file__))
     return [
-        f"{__name__}.{os.path.relpath(metric, curr_dir).replace('/', '.')[:-3]}"
-        for metric in glob.glob(f"{curr_dir}/**/*.py", recursive=True)
-        if os.path.isfile(metric) and os.path.basename(metric)[0].isupper()
+        f"{prefix}{test_id}" for test_id in vm_metrics_provider.list_tests(**kwargs)
     ]
 
 
-def load_metric(metric_id):
-    """Load a metric class from a string
-
-    Args:
-        metric_id (str): The metric id (e.g. 'validmind.unit_metrics.classification.F1')
-
-    Returns:
-        callable: The metric function
-    """
-    return getattr(import_module(metric_id), metric_id.split(".")[-1])
-
-
-def run_metric(metric_id, inputs=None, params=None, show=True, value_only=False):
-    """Run a single metric and cache the results
-
-    Args:
-        metric_id (str): The metric id (e.g. 'validmind.unit_metrics.classification.F1')
-        inputs (dict): A dictionary of the metric inputs
-        params (dict): A dictionary of the metric parameters
-        show (bool): Whether to display the results
-        value_only (bool): Whether to return only the value
-    """
-    inputs = {
-        k: input_registry.get(v) if isinstance(v, str) else v
-        for k, v in (inputs or {}).items()
-    }
-    params = params or {}
-
-    cache_key = _get_metric_cache_key(metric_id, inputs, params)
-
-    if cache_key not in unit_metric_results_cache:
-        metric = load_metric(metric_id)
-        _inputs, _params = _inspect_signature(metric)
-
-        result = metric(
-            **{k: v for k, v in inputs.items() if k in _inputs.keys()},
-            **{
-                k: v
-                for k, v in params.items()
-                if k in _params.keys() or "kwargs" in _params.keys()
-            },
-        )
-        unit_metric_results_cache[cache_key] = (
-            result,
-            # store the input ids that were used to calculate the result
-            [v.input_id for v in inputs.values()],
-            # store the params that were used to calculate the result
-            params,
-        )
-
-    cached_result = unit_metric_results_cache[cache_key]
+def describe_metric(metric_id: str, **kwargs):
+    """Describe a metric"""
+    return describe_test(metric_id, **kwargs)
 
-    if value_only:
-        return cached_result[0]
 
-    result_wrapper = _build_result(
-        results=cached_result[0],
-        test_id=metric_id,
-        inputs=cached_result[1],
-        params=cached_result[2],
-        generate_description=False,
-    )
+def run_metric(metric_id: str, **kwargs):
+    """Run a metric"""
+    return run_test(metric_id, **kwargs)
 
-    if show:
-        result_wrapper.show()
 
-    return result_wrapper
+__all__ = ["list_metrics", "describe_metric", "run_metric"]

validmind/unit_metrics/classification/F1.py

@@ -10,4 +10,5 @@ from validmind import tags, tasks
 @tasks("classification")
 @tags("classification")
 def F1(model, dataset, **kwargs):
+    """Calculates the F1 score for a classification model."""
     return f1_score(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/classification/Precision.py

@@ -10,4 +10,5 @@ from validmind import tags, tasks
 @tasks("classification")
 @tags("classification")
 def Precision(model, dataset, **kwargs):
+    """Calculates the precision for a classification model."""
     return precision_score(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/classification/ROC_AUC.py

@@ -12,6 +12,7 @@ from validmind import tags, tasks
 @tasks("classification")
 @tags("classification")
 def ROC_AUC(model, dataset, **kwargs):
+    """Calculates the ROC AUC for a classification model."""
     y_true = dataset.y
 
     if len(unique(y_true)) > 2:

validmind/unit_metrics/classification/Recall.py

@@ -10,4 +10,5 @@ from validmind import tags, tasks
 @tasks("classification")
 @tags("classification")
 def Recall(model, dataset, **kwargs):
+    """Calculates the recall for a classification model."""
     return recall_score(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/regression/AdjustedRSquaredScore.py

@@ -10,6 +10,7 @@ from validmind import tags, tasks
 @tags("regression")
 @tasks("regression")
 def AdjustedRSquaredScore(model, dataset):
+    """Calculates the adjusted R-squared score for a regression model."""
     r2_score = _r2_score(
         dataset.y,
         dataset.y_pred(model),

validmind/unit_metrics/regression/GiniCoefficient.py

@@ -10,6 +10,7 @@ from validmind import tags, tasks
 @tags("regression")
 @tasks("regression")
 def GiniCoefficient(dataset, model):
+    """Calculates the Gini coefficient for a regression model."""
     y_true = dataset.y
     y_pred = dataset.y_pred(model)
 

validmind/unit_metrics/regression/HuberLoss.py

@@ -10,6 +10,7 @@ from validmind import tags, tasks
 @tags("regression")
 @tasks("regression")
 def HuberLoss(model, dataset):
+    """Calculates the Huber loss for a regression model."""
     y_true = dataset.y
     y_pred = dataset.y_pred(model)
 

validmind/unit_metrics/regression/KolmogorovSmirnovStatistic.py

@@ -10,6 +10,7 @@ from validmind import tags, tasks
 @tags("regression")
 @tasks("regression")
 def KolmogorovSmirnovStatistic(dataset, model):
+    """Calculates the Kolmogorov-Smirnov statistic for a regression model."""
     y_true = dataset.y.flatten()
     y_pred = dataset.y_pred(model)
 

validmind/unit_metrics/regression/MeanAbsoluteError.py

@@ -10,4 +10,5 @@ from validmind import tags, tasks
 @tags("regression")
 @tasks("regression")
 def MeanAbsoluteError(model, dataset, **kwargs):
+    """Calculates the mean absolute error for a regression model."""
     return _mean_absolute_error(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/regression/MeanAbsolutePercentageError.py

@@ -10,6 +10,7 @@ from validmind import tags, tasks
 @tags("regression")
 @tasks("regression")
 def MeanAbsolutePercentageError(model, dataset):
+    """Calculates the mean absolute percentage error for a regression model."""
     y_true = dataset.y
     y_pred = dataset.y_pred(model)
 

validmind/unit_metrics/regression/MeanBiasDeviation.py

@@ -10,4 +10,5 @@ from validmind import tags, tasks
 @tags("regression")
 @tasks("regression")
 def MeanBiasDeviation(model, dataset):
+    """Calculates the mean bias deviation for a regression model."""
     return np.mean(dataset.y - dataset.y_pred(model))

validmind/unit_metrics/regression/MeanSquaredError.py

@@ -10,4 +10,5 @@ from validmind import tags, tasks
 @tags("regression")
 @tasks("regression")
 def MeanSquaredError(model, dataset, **kwargs):
+    """Calculates the mean squared error for a regression model."""
     return mean_squared_error(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/regression/QuantileLoss.py

@@ -10,6 +10,7 @@ from validmind import tags, tasks
 @tags("regression")
 @tasks("regression")
 def QuantileLoss(model, dataset, quantile=0.5):
+    """Calculates the quantile loss for a regression model."""
     error = dataset.y - dataset.y_pred(model)
 
     return np.mean(np.maximum(quantile * error, (quantile - 1) * error))

validmind/unit_metrics/regression/RSquaredScore.py

@@ -9,5 +9,6 @@ from validmind import tags, tasks
 
 @tags("regression")
 @tasks("regression")
-def RSquaredError(model, dataset):
+def RSquaredScore(model, dataset):
+    """Calculates the R-squared score for a regression model."""
     return r2_score(dataset.y, dataset.y_pred(model))

validmind/unit_metrics/regression/RootMeanSquaredError.py

@@ -11,6 +11,7 @@ from validmind import tags, tasks
 @tags("regression")
 @tasks("regression")
 def RootMeanSquaredError(model, dataset, **kwargs):
+    """Calculates the root mean squared error for a regression model."""
     return np.sqrt(
         mean_squared_error(
             dataset.y,