validmind 2.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl

validmind/errors.py

@@ -48,7 +48,7 @@ class MissingCacheResultsArgumentsError(BaseError):
     pass
 
 
-class MissingModelPredictFnError(BaseError):
+class MissingOrInvalidModelPredictFnError(BaseError):
     """
     When the pytorch model is missing a predict function or its predict
     method does not have the expected arguments.
@@ -315,6 +315,14 @@ class UnsupportedModelError(BaseError):
     pass
 
 
+class UnsupportedModelForSHAPError(BaseError):
+    """
+    When an unsupported model is used for SHAP importance.
+    """
+
+    pass
+
+
 class SkipTestError(BaseError):
     """
     Useful error to throw when a test cannot be executed.
@@ -361,6 +369,8 @@ def should_raise_on_fail_fast(error) -> bool:
     """
     error_class = error.__class__.__name__
     return error_class not in [
+        "MissingOrInvalidModelPredictFnError",
         "MissingRequiredTestInputError",
         "SkipTestError",
+        "UnsupportedModelForSHAPError",
     ]

validmind/logging.py

@@ -68,10 +68,17 @@ def get_logger(name="validmind", log_level=None):
     logger = logging.getLogger(name)
     logger.setLevel(log_level or _get_log_level())
 
-    # Check if the handler is already added
-    if not any(isinstance(h, type(handler)) for h in logger.handlers):
+    # Clear existing handlers if any (or refine the existing logic as necessary)
+    # TODO: lets add some better handler management
+    if not any(
+        isinstance(h, type(handler)) and h.formatter._fmt == formatter._fmt
+        for h in logger.handlers
+    ):
         logger.addHandler(handler)
 
+    # Prevent logger from propagating to root logger
+    logger.propagate = False
+
     return logger
 
 

validmind/models/huggingface.py

@@ -6,7 +6,7 @@ from dataclasses import dataclass
 
 import pandas as pd
 
-from validmind.errors import MissingModelPredictFnError
+from validmind.errors import MissingOrInvalidModelPredictFnError
 from validmind.logging import get_logger
 from validmind.vm_models.model import (
     ModelAttributes,
@@ -44,7 +44,7 @@ class HFModel(VMModel):
         Invoke predict_proba from underline model
         """
         if not has_method_with_arguments(self.model, "predict_proba", 1):
-            raise MissingModelPredictFnError(
+            raise MissingOrInvalidModelPredictFnError(
                 "Model requires a implementation of predict_proba method with 1 argument"
                 + " that is tensor features matrix"
             )

validmind/models/pytorch.py

@@ -2,7 +2,7 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from validmind.errors import MissingModelPredictFnError
+from validmind.errors import MissingOrInvalidModelPredictFnError
 from validmind.logging import get_logger
 from validmind.vm_models.model import (
     ModelAttributes,
@@ -41,7 +41,7 @@ class PyTorchModel(VMModel):
         Invoke predict_proba from underline model
         """
         if not has_method_with_arguments(self.model, "predict_proba", 1):
-            raise MissingModelPredictFnError(
+            raise MissingOrInvalidModelPredictFnError(
                 "Model requires a implemention of predict_proba method with 1 argument"
                 + " that is tensor features matrix"
             )
@@ -54,7 +54,7 @@ class PyTorchModel(VMModel):
         Predict method for the model. This is a wrapper around the model's
         """
         if not has_method_with_arguments(self.model, "predict", 1):
-            raise MissingModelPredictFnError(
+            raise MissingOrInvalidModelPredictFnError(
                 "Model requires a implemention of predict method with 1 argument"
                 + " that is tensor features matrix"
             )

validmind/models/sklearn.py

@@ -2,7 +2,7 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from validmind.errors import MissingModelPredictFnError
+from validmind.errors import MissingOrInvalidModelPredictFnError
 from validmind.logging import get_logger
 from validmind.vm_models.model import (
     ModelAttributes,
@@ -40,9 +40,9 @@ class SKlearnModel(VMModel):
         predict_proba (for classification) or predict (for regression) method
         """
         if not has_method_with_arguments(self.model, "predict_proba", 1):
-            raise MissingModelPredictFnError(
-                "Model requires a implemention of predict_proba method with 1 argument"
-                + " that is features matrix"
+            raise MissingOrInvalidModelPredictFnError(
+                f"SKlearn model {self.model.__class__} Model does not have a compatible predict_proba implementation."
+                + " Please assign predictions directly with vm_dataset.assign_predictions(model, prediction_values)"
             )
         if callable(getattr(self.model, "predict_proba", None)):
             return self.model.predict_proba(*args, **kwargs)[:, 1]

validmind/template.py

@@ -4,9 +4,9 @@
 
 from pprint import pformat
 
+import mistune
 from IPython.display import display
 from ipywidgets import HTML, Accordion, VBox
-from markdown import markdown
 
 from .html_templates.content_blocks import (
     failed_content_block_html,
@@ -75,7 +75,7 @@ def _create_content_widget(content):
             HTML(
                 test_content_block_html.format(
                     title=test_deets["Name"],
-                    description=markdown(test_deets["Description"]),
+                    description=mistune.html(test_deets["Description"]),
                     required_inputs=", ".join(
                         test_deets["Required Inputs"] or ["None"]
                     ),

validmind/test_suites/__init__.py

@@ -5,6 +5,8 @@
 """
 Entrypoint for test suites.
 """
+from inspect import getdoc
+
 import pandas as pd
 
 from ..logging import get_logger
@@ -139,7 +141,7 @@ def list_suites(pretty: bool = True):
             {
                 "ID": suite_id,
                 "Name": test_suite.__name__,
-                "Description": test_suite.__doc__.strip(),
+                "Description": getdoc(test_suite).strip(),
                 "Tests": ", ".join(_get_test_suite_test_ids(test_suite)),
             }
         )
@@ -167,7 +169,7 @@ def describe_suite(test_suite_id: str, verbose=False):
                     {
                         "ID": test_suite_id,
                         "Name": test_suite.__name__,
-                        "Description": test_suite.__doc__.strip(),
+                        "Description": getdoc(test_suite).strip(),
                         "Tests": ", ".join(_get_test_suite_test_ids(test_suite)),
                     }
                 ]

validmind/tests/__init__.py

@@ -5,23 +5,26 @@
 """All Tests for ValidMind"""
 
 import importlib
+import inspect
 import sys
 from pathlib import Path
 from pprint import pformat
 from typing import Dict
 
+import mistune
 import pandas as pd
 from IPython.display import display
 from ipywidgets import HTML
-from markdown import markdown
 
 from ..errors import LoadTestError
 from ..html_templates.content_blocks import test_content_block_html
 from ..logging import get_logger
-from ..utils import clean_docstring, format_dataframe, fuzzy_match, test_id_to_name
+from ..unit_metrics import run_metric
+from ..unit_metrics.composite import load_composite_metric
+from ..utils import format_dataframe, fuzzy_match, test_id_to_name
 from ..vm_models import TestContext, TestInput
-from .__types__ import ExternalTestProvider
-from .test_providers import GithubTestProvider, LocalTestProvider
+from .decorator import metric, tags, tasks
+from .test_providers import LocalTestProvider, TestProvider
 
 logger = get_logger(__name__)
 
@@ -34,22 +37,28 @@ __all__ = [
     "load_test",
     "describe_test",
     "register_test_provider",
-    "GithubTestProvider",
     "LoadTestError",
     "LocalTestProvider",
+    # Decorators for functional metrics
+    "metric",
+    "tags",
+    "tasks",
 ]
 
 __tests = None
 __test_classes = None
 
-__test_providers: Dict[str, ExternalTestProvider] = {}
+__test_providers: Dict[str, TestProvider] = {}
+__custom_tests: Dict[str, object] = {}
 
 
 def _test_description(test_class, truncate=True):
-    if truncate and len(test_class.__doc__.split("\n")) > 5:
-        return test_class.__doc__.strip().split("\n")[0] + "..."
+    description = inspect.getdoc(test_class).strip()
 
-    return test_class.__doc__
+    if truncate and len(description.split("\n")) > 5:
+        return description.strip().split("\n")[0] + "..."
+
+    return description
 
 
 def _load_tests(test_ids):
@@ -249,55 +258,83 @@ def list_tests(filter=None, task=None, tags=None, pretty=True, truncate=True):
     return tests
 
 
-def load_test(test_id, reload=False):  # noqa: C901
-    # Extract the test ID extension from the actual test ID when loading
-    # the test class. This enables us to generate multiple results for
-    # the same tests within the document. For instance, consider the
-    # test ID "validmind.data_validation.ClassImbalance:data_id_1,"
-    # where the test ID extension is "data_id_1".
+def _load_validmind_test(test_id, reload=False):
     parts = test_id.split(":")[0].split(".")
 
+    test_module = ".".join(parts[1:-1])
+    test_class = parts[-1]
+
     error = None
-    namespace = parts[0]
+    test = None
 
-    if namespace != "validmind" and namespace not in __test_providers:
-        error = (
-            f"Unable to load test {test_id}. "
-            f"No Test Provider found for the namespace: {namespace}."
-        )
+    try:
+        full_path = f"validmind.tests.{test_module}.{test_class}"
 
-    if namespace == "validmind":
-        test_module = ".".join(parts[1:-1])
-        test_class = parts[-1]
+        if reload and full_path in sys.modules:
+            module = importlib.reload(sys.modules[full_path])
+        else:
+            module = importlib.import_module(full_path)
 
-        try:
-            full_path = f"validmind.tests.{test_module}.{test_class}"
+        test = getattr(module, test_class)
+    except ModuleNotFoundError as e:
+        error = f"Unable to load test {test_id}. {e}"
+    except AttributeError:
+        error = f"Unable to load test {test_id}. Test not in module: {test_class}"
+
+    return error, test
+
+
+def load_test(test_id: str, reload=False):
+    """Load a test by test ID
+
+    Test IDs are in the format `namespace.path_to_module.TestClassOrFuncName[:result_id]`.
+    The result ID is optional and is used to distinguish between multiple results from the
+    running the same test.
+
+    Args:
+        test_id (str): The test ID in the format `namespace.path_to_module.TestName[:result_id]`
+        reload (bool, optional): Whether to reload the test module. Defaults to False.
+    """
+    # TODO: we should use a dedicated class for test IDs to handle this consistently
+    test_id, result_id = test_id.split(":", 1) if ":" in test_id else (test_id, None)
+
+    error = None
+    namespace = test_id.split(".", 1)[0]
+
+    # TODO: lets implement an extensible loading system instead of this ugly if/else
+    if test_id in __custom_tests:
+        test = __custom_tests[test_id]
 
-            if reload and full_path in sys.modules:
-                module = importlib.reload(sys.modules[full_path])
-            else:
-                module = importlib.import_module(full_path)
+    elif test_id.startswith("validmind.composite_metric"):
+        error, test = load_composite_metric(test_id)
 
-            test = getattr(module, test_class)
-        except ModuleNotFoundError as e:
-            error = f"Unable to load test {test_id}. {e}"
-        except AttributeError:
-            error = f"Unable to load test {test_id}. Class not in module: {test_class}"
+    elif namespace == "validmind":
+        error, test = _load_validmind_test(test_id, reload=reload)
 
     elif namespace in __test_providers:
         try:
             test = __test_providers[namespace].load_test(test_id.split(".", 1)[1])
         except Exception as e:
             error = (
-                f"Unable to load test {test_id} from test  provider: "
+                f"Unable to load test {test_id} from test provider: "
                 f"{__test_providers[namespace]}\n Got Exception: {e}"
             )
 
+    else:
+        error = f"Unable to load test {test_id}. No test provider found."
+
     if error:
         logger.error(error)
         raise LoadTestError(error)
 
-    test.test_id = test_id
+    if inspect.isfunction(test):
+        # if its a function, we decorate it and then load the class
+        # TODO: simplify this as we move towards all functional metrics
+        # "_" is used here so it doesn't conflict with other test ids
+        metric("_")(test)
+        test = __custom_tests["_"]
+
+    test.test_id = f"{test_id}:{result_id}" if result_id else test_id
 
     return test
 
@@ -322,7 +359,7 @@ def describe_test(test_id: str = None, raw: bool = False):
         "Test Type": test.test_type,
         "Required Inputs": test.required_inputs,
         "Params": test.default_params or {},
-        "Description": clean_docstring(test.__doc__),
+        "Description": inspect.getdoc(test).strip() or "",
     }
 
     if raw:
@@ -332,7 +369,7 @@ def describe_test(test_id: str = None, raw: bool = False):
         HTML(
             test_content_block_html.format(
                 title=f'{details["Name"]}',
-                description=markdown(details["Description"]),
+                description=mistune.html(details["Description"].strip()),
                 required_inputs=", ".join(details["Required Inputs"] or ["None"]),
                 params_table="\n".join(
                     [
@@ -346,21 +383,63 @@ def describe_test(test_id: str = None, raw: bool = False):
     )
 
 
-def run_test(test_id, params: dict = None, inputs=None, output_template=None, **kwargs):
+def run_test(
+    test_id: str = None,
+    name: str = None,
+    unit_metrics: list = None,
+    params: dict = None,
+    inputs=None,
+    output_template=None,
+    show=True,
+    **kwargs,
+):
     """Run a test by test ID
 
     Args:
-        test_id (str): The test ID
+        test_id (str, option): The test ID to run - required when running a single test
+            i.e. when not running multiple unit metrics
+        name (str, optional): The name of the test (used to create a composite metric
+            out of multiple unit metrics) - required when running multiple unit metrics
+        unit_metrics (list, optional): A list of unit metric IDs to run as a composite
+            metric - required when running multiple unit metrics
         params (dict, optional): A dictionary of params to override the default params
         inputs: A dictionary of test inputs to pass to the Test
         output_template (str, optional): A template to use for customizing the output
+        show (bool, optional): Whether to display the results. Defaults to True.
         **kwargs: Any extra arguments will be passed in via the TestInput object. i.e.:
             - dataset: A validmind Dataset object or a Pandas DataFrame
             - model: A model to use for the test
             - models: A list of models to use for the test
             other inputs can be accessed inside the test via `self.inputs["input_name"]`
     """
-    TestClass = load_test(test_id, reload=True)
+    if not test_id and not name and not unit_metrics:
+        raise ValueError(
+            "`test_id` or `name` and `unit_metrics` must be provided to run a test"
+        )
+
+    if (unit_metrics and not name) or (name and not unit_metrics):
+        raise ValueError("`name` and `unit_metrics` must be provided together")
+
+    if test_id and test_id.startswith("validmind.unit_metrics"):
+        # TODO: as we move towards a more unified approach to metrics
+        # we will want to make everything functional and remove the
+        # separation between unit metrics and "normal" metrics
+        return run_metric(test_id, inputs=inputs, params=params, show=show)
+
+    if unit_metrics:
+        metric_id_name = "".join(word[0].upper() + word[1:] for word in name.split())
+        test_id = f"validmind.composite_metric.{metric_id_name}"
+
+        error, TestClass = load_composite_metric(
+            unit_metrics=unit_metrics, metric_name=metric_id_name
+        )
+
+        if error:
+            raise LoadTestError(error)
+
+    else:
+        TestClass = load_test(test_id, reload=True)
+
     test = TestClass(
         test_id=test_id,
         context=TestContext(),
@@ -370,16 +449,22 @@ def run_test(test_id, params: dict = None, inputs=None, output_template=None, **
     )
 
     test.run()
-    test.result.show()
+
+    if show:
+        test.result.show()
 
     return test.result
 
 
-def register_test_provider(namespace: str, test_provider: ExternalTestProvider) -> None:
+def register_test_provider(namespace: str, test_provider: TestProvider) -> None:
     """Register an external test provider
 
     Args:
         namespace (str): The namespace of the test provider
-        test_provider (ExternalTestProvider): The test provider
+        test_provider (TestProvider): The test provider
     """
     __test_providers[namespace] = test_provider
+
+
+def _register_custom_test(test_id: str, test_class: object):
+    __custom_tests[test_id] = test_class

validmind/tests/data_validation/DatasetDescription.py

@@ -122,7 +122,6 @@ class DatasetDescription(Metric):
         return self.cache_results(results)
 
     def infer_datatype(self, df):
-
         vm_dataset_variables = {}
         typeset = ProfilingTypeSet(Settings())
         variable_types = typeset.infer_type(df)

validmind/tests/data_validation/FeatureTargetCorrelationPlot.py

@@ -74,7 +74,9 @@ class FeatureTargetCorrelationPlot(Metric):
 
     def visualize_feature_target_correlation(self, df, target_column, fig_height):
         # Compute correlations with the target variable
-        correlations = df.corr(numeric_only=True)[target_column].drop(target_column)
+        correlations = (
+            df.corr(numeric_only=True)[target_column].drop(target_column).to_frame()
+        )
         correlations = correlations.loc[:, ~correlations.columns.duplicated()]
 
         correlations = correlations.sort_values(by=target_column, ascending=True)

validmind/tests/data_validation/PiTCreditScoresHistogram.py

@@ -113,7 +113,7 @@ class PiTCreditScoresHistogram(Metric):
         )
         predicted_default_column = (
             self.params.get("predicted_default_column")
-            or self.inputs.dataset.y_pred(self.inputs.model.input_id),
+            or self.inputs.dataset.y_pred(self.inputs.model),
         )
         scores_column = self.params["scores_column"]
         point_in_time_column = self.params["point_in_time_column"]

validmind/tests/data_validation/ScatterPlot.py

@@ -65,8 +65,14 @@ class ScatterPlot(Metric):
         if not set(columns).issubset(set(df.columns)):
             raise ValueError("Provided 'columns' must exist in the dataset")
 
-        sns.pairplot(data=df, diag_kind="kde")
-
+        g = sns.pairplot(data=df, diag_kind="kde")
+        for ax in g.axes.flatten():
+            # rotate x axis labels
+            ax.set_xlabel(ax.get_xlabel(), rotation=45)
+            # rotate y axis labels
+            ax.set_ylabel(ax.get_ylabel(), rotation=45)
+            # set y labels alignment
+            ax.yaxis.get_label().set_horizontalalignment("right")
         # Get the current figure
         fig = plt.gcf()
 

validmind/tests/data_validation/nlp/StopWords.py

@@ -22,7 +22,6 @@ from validmind.vm_models import (
     ResultTableMetadata,
     ThresholdTest,
     ThresholdTestResult,
-    VMDataset,
 )
 
 
@@ -86,17 +85,13 @@ class StopWords(ThresholdTest):
                 ResultTable(
                     data=df,
                     metadata=ResultTableMetadata(
-                        title=f"Class Imbalance Results for Column {self.inputs.dataset.target_column}"
+                        title=f"Stop words results for column '{self.inputs.dataset.target_column}'"
                     ),
                 )
             ]
         )
 
     def run(self):
-        # Can only run this test if we have a Dataset object
-        if not isinstance(self.inputs.dataset, VMDataset):
-            raise ValueError("ClassImbalance requires a validmind Dataset object")
-
         text_column = self.inputs.dataset.text_column
 
         def create_corpus(df, text_column):

validmind/tests/data_validation/nlp/TextDescription.py

@@ -92,9 +92,12 @@ class TextDescription(Metric):
             total_words = len(words)
             total_sentences = len(sentences)
             avg_sentence_length = round(
-                sum(len(sentence.split()) for sentence in sentences) / total_sentences
-                if total_sentences
-                else 0,
+                (
+                    sum(len(sentence.split()) for sentence in sentences)
+                    / total_sentences
+                    if total_sentences
+                    else 0
+                ),
                 1,
             )
             total_paragraphs = len(paragraphs)
@@ -161,9 +164,13 @@ class TextDescription(Metric):
         return combined_df
 
     def run(self):
+        # Enforce that text_column must be provided as part of the params
+        if self.inputs.dataset.text_column is None:
+            raise ValueError("A 'text_column' must be provided to run this test.")
+
         # Can only run this test if we have a Dataset object
         if not isinstance(self.inputs.dataset, VMDataset):
-            raise ValueError("TextDescretion requires a validmind Dataset object")
+            raise ValueError("TextDescription requires a validmind Dataset object")
 
         df_text_description = self.text_description_table(
             self.inputs.dataset.df, self.params
@@ -177,27 +184,31 @@ class TextDescription(Metric):
             ("Total Unique Words", "Lexical Diversity"),
         ]
         params = {"combinations_to_plot": combinations_to_plot}
-        figures = self.text_description_scatter_plot(df_text_description, params)
+        figures = self.text_description_plots(df_text_description, params)
 
         return self.cache_results(
             figures=figures,
         )
 
     # Function to plot scatter plots for specified combinations using Plotly
-    def text_description_scatter_plot(self, df, params):
+    def text_description_plots(self, df, params):
         combinations_to_plot = params["combinations_to_plot"]
         figures = []
         # Create hist plots for each column
         for i, column in enumerate(df.columns):
             fig = px.histogram(df, x=column)
             fig.update_layout(bargap=0.2)
-            figures.append(Figure(for_object=self, key=self.key, figure=fig))
+            # Generate a unique key for each histogram using the column name and index
+            histogram_key = f"{self.name}_histogram_{column}_{i}"
+            figures.append(Figure(for_object=self, key=histogram_key, figure=fig))
 
-        for metric1, metric2 in combinations_to_plot:
+        for j, (metric1, metric2) in enumerate(combinations_to_plot):
             fig = px.scatter(
                 df, x=metric1, y=metric2, title=f"Scatter Plot: {metric1} vs {metric2}"
             )
-            figures.append(Figure(for_object=self, key=self.key, figure=fig))
+            # Generate a unique key for each scatter plot using the metric names and index
+            scatter_key = f"{self.name}_scatter_{metric1}_vs_{metric2}_{j}"
+            figures.append(Figure(for_object=self, key=scatter_key, figure=fig))
         plt.close("all")
 
         return figures