validmind 2.8.12__py3-none-any.whl → 2.8.22__py3-none-any.whl

validmind/tests/data_validation/ClassImbalance.py

@@ -14,7 +14,9 @@ from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset
 
 
-@tags("tabular_data", "binary_classification", "multiclass_classification")
+@tags(
+    "tabular_data", "binary_classification", "multiclass_classification", "data_quality"
+)
 @tasks("classification")
 def ClassImbalance(
     dataset: VMDataset, min_percent_threshold: int = 10

validmind/tests/data_validation/DatasetDescription.py

@@ -6,12 +6,10 @@ import re
 from collections import Counter
 
 import numpy as np
-from ydata_profiling.config import Settings
-from ydata_profiling.model.typeset import ProfilingTypeSet
 
 from validmind import RawData, tags, tasks
-from validmind.errors import UnsupportedColumnTypeError
 from validmind.logging import get_logger
+from validmind.utils import infer_datatypes
 from validmind.vm_models import VMDataset
 
 DEFAULT_HISTOGRAM_BINS = 10
@@ -20,25 +18,6 @@ DEFAULT_HISTOGRAM_BIN_SIZES = [5, 10, 20, 50]
 logger = get_logger(__name__)
 
 
-def infer_datatypes(df):
-    column_type_mappings = {}
-    typeset = ProfilingTypeSet(Settings())
-    variable_types = typeset.infer_type(df)
-
-    for column, type in variable_types.items():
-        if str(type) == "Unsupported":
-            if df[column].isnull().all():
-                column_type_mappings[column] = {"id": column, "type": "Null"}
-            else:
-                raise UnsupportedColumnTypeError(
-                    f"Unsupported type for column {column}. Please review all values in this dataset column."
-                )
-        else:
-            column_type_mappings[column] = {"id": column, "type": str(type)}
-
-    return list(column_type_mappings.values())
-
-
 def get_numerical_histograms(df, column):
     """
     Returns a collection of histograms for a numerical column, each one
@@ -50,7 +29,7 @@ def get_numerical_histograms(df, column):
     # bins='sturges'. Cannot use 'auto' until we review and fix its performance
     #  on datasets with too many unique values
     #
-    # 'sturges': R’s default method, only accounts for data size. Only optimal
+    # 'sturges': R's default method, only accounts for data size. Only optimal
     # for gaussian data and underestimates number of bins for large non-gaussian datasets.
     default_hist = np.histogram(values_cleaned, bins="sturges")
 

validmind/tests/data_validation/DescriptiveStatistics.py

@@ -44,7 +44,7 @@ def get_summary_statistics_categorical(df, categorical_fields):
     return summary_stats
 
 
-@tags("tabular_data", "time_series_data")
+@tags("tabular_data", "time_series_data", "data_quality")
 @tasks("classification", "regression")
 def DescriptiveStatistics(dataset: VMDataset):
     """

validmind/tests/data_validation/Skewness.py

@@ -2,10 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from ydata_profiling.config import Settings
-from ydata_profiling.model.typeset import ProfilingTypeSet
-
 from validmind import tags, tasks
+from validmind.utils import infer_datatypes
 
 
 @tags("data_quality", "tabular_data")
@@ -49,8 +47,11 @@ def Skewness(dataset, max_threshold=1):
     - Subjective threshold for risk grading, requiring expert input and recurrent iterations for refinement.
     """
 
-    typeset = ProfilingTypeSet(Settings())
-    dataset_types = typeset.infer_type(dataset.df)
+    # Use the imported infer_datatypes function
+    dataset_types = infer_datatypes(dataset.df)
+
+    # Convert the list of dictionaries to a dictionary for easy access
+    dataset_types_dict = {item["id"]: item["type"] for item in dataset_types}
 
     skewness = dataset.df.skew(numeric_only=True)
 
@@ -58,7 +59,7 @@ def Skewness(dataset, max_threshold=1):
     passed = True
 
     for col in skewness.index:
-        if str(dataset_types[col]) != "Numeric":
+        if dataset_types_dict.get(col) != "Numeric":
             continue
 
         col_skewness = skewness[col]

validmind/tests/decorator.py

@@ -7,6 +7,7 @@
 import inspect
 import os
 from functools import wraps
+from typing import Any, Callable, List, Optional, TypeVar, Union
 
 from validmind.logging import get_logger
 
@@ -15,8 +16,10 @@ from .load import load_test
 
 logger = get_logger(__name__)
 
+F = TypeVar("F", bound=Callable[..., Any])
 
-def _get_save_func(func, test_id):
+
+def _get_save_func(func: Callable[..., Any], test_id: str) -> Callable[..., None]:
     """Helper function to save a decorated function to a file
 
     Useful when a custom test function has been created inline in a notebook or
@@ -29,7 +32,7 @@ def _get_save_func(func, test_id):
     # remove decorator line
     source = source.split("\n", 1)[1]
 
-    def save(root_folder=".", imports=None):
+    def save(root_folder: str = ".", imports: Optional[List[str]] = None) -> None:
         parts = test_id.split(".")
 
         if len(parts) > 1:
@@ -84,7 +87,7 @@ def _get_save_func(func, test_id):
     return save
 
 
-def test(func_or_id):
+def test(func_or_id: Union[Callable[..., Any], str, None]) -> Callable[[F], F]:
     """Decorator for creating and registering custom tests
 
     This decorator registers the function it wraps as a test function within ValidMind
@@ -109,14 +112,14 @@ def test(func_or_id):
     as the metric's description.
 
     Args:
-        func: The function to decorate
-        test_id: The identifier for the metric. If not provided, the function name is used.
+        func_or_id (Union[Callable[..., Any], str, None]): Either the function to decorate
+            or the test ID. If None, the function name is used.
 
     Returns:
-        The decorated function.
+        Callable[[F], F]: The decorated function.
     """
 
-    def decorator(func):
+    def decorator(func: F) -> F:
         test_id = func_or_id or f"validmind.custom_metrics.{func.__name__}"
         test_func = load_test(test_id, func, reload=True)
         test_store.register_test(test_id, test_func)
@@ -136,28 +139,28 @@ def test(func_or_id):
     return decorator
 
 
-def tasks(*tasks):
+def tasks(*tasks: str) -> Callable[[F], F]:
     """Decorator for specifying the task types that a test is designed for.
 
     Args:
         *tasks: The task types that the test is designed for.
     """
 
-    def decorator(func):
+    def decorator(func: F) -> F:
         func.__tasks__ = list(tasks)
         return func
 
     return decorator
 
 
-def tags(*tags):
+def tags(*tags: str) -> Callable[[F], F]:
     """Decorator for specifying tags for a test.
 
     Args:
         *tags: The tags to apply to the test.
     """
 
-    def decorator(func):
+    def decorator(func: F) -> F:
         func.__tags__ = list(tags)
         return func
 

validmind/tests/load.py

@@ -7,7 +7,7 @@
 import inspect
 import json
 from pprint import pformat
-from typing import List
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 from uuid import uuid4
 
 import pandas as pd
@@ -32,7 +32,10 @@ INPUT_TYPE_MAP = {
 }
 
 
-def _inspect_signature(test_func: callable):
+def _inspect_signature(
+    test_func: Callable[..., Any],
+) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, Dict[str, Any]]]:
+    """Inspect a test function's signature to get inputs and parameters"""
     inputs = {}
     params = {}
 
@@ -56,7 +59,9 @@ def _inspect_signature(test_func: callable):
     return inputs, params
 
 
-def load_test(test_id: str, test_func: callable = None, reload: bool = False):
+def load_test(
+    test_id: str, test_func: Optional[Callable[..., Any]] = None, reload: bool = False
+) -> Callable[..., Any]:
     """Load a test by test ID
 
     Test IDs are in the format `namespace.path_to_module.TestClassOrFuncName[:tag]`.
@@ -67,6 +72,8 @@ def load_test(test_id: str, test_func: callable = None, reload: bool = False):
         test_id (str): The test ID in the format `namespace.path_to_module.TestName[:tag]`
         test_func (callable, optional): The test function to load. If not provided, the
             test will be loaded from the test provider. Defaults to None.
+        reload (bool, optional): If True, reload the test even if it's already loaded.
+            Defaults to False.
     """
     # remove tag if present
     test_id = test_id.split(":", 1)[0]
@@ -109,7 +116,8 @@ def load_test(test_id: str, test_func: callable = None, reload: bool = False):
     return test_store.get_test(test_id)
 
 
-def _list_test_ids():
+def _list_test_ids() -> List[str]:
+    """List all available test IDs"""
     test_ids = []
 
     for namespace, test_provider in test_provider_store.test_providers.items():
@@ -120,7 +128,7 @@ def _list_test_ids():
     return test_ids
 
 
-def _load_tests(test_ids):
+def _load_tests(test_ids: List[str]) -> Dict[str, Callable[..., Any]]:
     """Load a set of tests, handling missing dependencies."""
     tests = {}
 
@@ -138,12 +146,12 @@ def _load_tests(test_ids):
             logger.debug(str(e))
 
             if e.extra:
-                logger.info(
+                logger.debug(
                     f"Skipping `{test_id}` as it requires extra dependencies: {e.required_dependencies}."
                     f" Please run `pip install validmind[{e.extra}]` to view and run this test."
                 )
             else:
-                logger.info(
+                logger.debug(
                     f"Skipping `{test_id}` as it requires missing dependencies: {e.required_dependencies}."
                     " Please install the missing dependencies to view and run this test."
                 )
@@ -151,7 +159,8 @@ def _load_tests(test_ids):
     return tests
 
 
-def _test_description(test_description: str, num_lines: int = 5):
+def _test_description(test_description: str, num_lines: int = 5) -> str:
+    """Format a test description"""
     description = test_description.strip("\n").strip()
 
     if len(description.split("\n")) > num_lines:
@@ -160,7 +169,10 @@ def _test_description(test_description: str, num_lines: int = 5):
     return description
 
 
-def _pretty_list_tests(tests, truncate=True):
+def _pretty_list_tests(
+    tests: Dict[str, Callable[..., Any]], truncate: bool = True
+) -> None:
+    """Pretty print a list of tests"""
     table = [
         {
             "ID": test_id,
@@ -171,6 +183,8 @@ def _pretty_list_tests(tests, truncate=True):
             ),
             "Required Inputs": list(test.inputs.keys()),
             "Params": test.params,
+            "Tags": test.__tags__,
+            "Tasks": test.__tasks__,
         }
         for test_id, test in tests.items()
     ]
@@ -178,10 +192,8 @@ def _pretty_list_tests(tests, truncate=True):
     return format_dataframe(pd.DataFrame(table))
 
 
-def list_tags():
-    """
-    List unique tags from all test classes.
-    """
+def list_tags() -> List[str]:
+    """List all unique available tags"""
 
     unique_tags = set()
 
@@ -191,7 +203,7 @@ def list_tags():
     return list(unique_tags)
 
 
-def list_tasks_and_tags(as_json=False):
+def list_tasks_and_tags(as_json: bool = False) -> Union[str, Dict[str, List[str]]]:
     """
     List all task types and their associated tags, with one row per task type and
     all tags for a task type in one row.
@@ -218,11 +230,8 @@ def list_tasks_and_tags(as_json=False):
     )
 
 
-def list_tasks():
-    """
-    List unique tasks from all test classes.
-    """
-
+def list_tasks() -> List[str]:
+    """List all unique available tasks"""
     unique_tasks = set()
 
     for test in _load_tests(list_tests(pretty=False)).values():
@@ -231,7 +240,13 @@ def list_tasks():
     return list(unique_tasks)
 
 
-def list_tests(filter=None, task=None, tags=None, pretty=True, truncate=True):
+def list_tests(
+    filter: Optional[str] = None,
+    task: Optional[str] = None,
+    tags: Optional[List[str]] = None,
+    pretty: bool = True,
+    truncate: bool = True,
+) -> Union[List[str], None]:
     """List all tests in the tests directory.
 
     Args:
@@ -245,9 +260,6 @@ def list_tests(filter=None, task=None, tags=None, pretty=True, truncate=True):
             formatted table. Defaults to True.
         truncate (bool, optional): If True, truncates the test description to the first
             line. Defaults to True. (only used if pretty=True)
-
-    Returns:
-        list or pandas.DataFrame: A list of all tests or a formatted table.
     """
     test_ids = _list_test_ids()
 
@@ -286,7 +298,9 @@ def list_tests(filter=None, task=None, tags=None, pretty=True, truncate=True):
     return _pretty_list_tests(tests, truncate=truncate)
 
 
-def describe_test(test_id: TestID = None, raw: bool = False, show: bool = True):
+def describe_test(
+    test_id: Optional[TestID] = None, raw: bool = False, show: bool = True
+) -> Union[str, HTML, Dict[str, Any]]:
     """Get or show details about the test
 
     This function can be used to see test details including the test name, description,

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -123,8 +123,10 @@ def AnswerCorrectness(
 
     score_column = "answer_correctness"
 
-    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
-    fig_box = px.box(x=result_df[score_column].to_list())
+    fig_histogram = px.histogram(
+        x=result_df[score_column].to_list(), nbins=10, title="Answer Correctness"
+    )
+    fig_box = px.box(x=result_df[score_column].to_list(), title="Answer Correctness")
 
     return (
         {

validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -118,8 +118,10 @@ def ContextEntityRecall(
 
     score_column = "context_entity_recall"
 
-    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
-    fig_box = px.box(x=result_df[score_column].to_list())
+    fig_histogram = px.histogram(
+        x=result_df[score_column].to_list(), nbins=10, title="Context Entity Recall"
+    )
+    fig_box = px.box(x=result_df[score_column].to_list(), title="Context Entity Recall")
 
     return (
         {

validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -114,8 +114,10 @@ def ContextPrecision(
 
     score_column = "llm_context_precision_with_reference"
 
-    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
-    fig_box = px.box(x=result_df[score_column].to_list())
+    fig_histogram = px.histogram(
+        x=result_df[score_column].to_list(), nbins=10, title="Context Precision"
+    )
+    fig_box = px.box(x=result_df[score_column].to_list(), title="Context Precision")
 
     return (
         {

validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py

@@ -109,8 +109,10 @@ def ContextPrecisionWithoutReference(
 
     score_column = "llm_context_precision_without_reference"
 
-    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
-    fig_box = px.box(x=result_df[score_column].to_list())
+    fig_histogram = px.histogram(
+        x=result_df[score_column].to_list(), nbins=10, title="Context Precision"
+    )
+    fig_box = px.box(x=result_df[score_column].to_list(), title="Context Precision")
 
     return (
         {

validmind/tests/model_validation/ragas/ContextRecall.py

@@ -114,8 +114,10 @@ def ContextRecall(
 
     score_column = "context_recall"
 
-    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
-    fig_box = px.box(x=result_df[score_column].to_list())
+    fig_histogram = px.histogram(
+        x=result_df[score_column].to_list(), nbins=10, title="Context Recall"
+    )
+    fig_box = px.box(x=result_df[score_column].to_list(), title="Context Recall")
 
     return (
         {

validmind/tests/model_validation/ragas/Faithfulness.py

@@ -119,8 +119,10 @@ def Faithfulness(
 
     score_column = "faithfulness"
 
-    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
-    fig_box = px.box(x=result_df[score_column].to_list())
+    fig_histogram = px.histogram(
+        x=result_df[score_column].to_list(), nbins=10, title="Faithfulness"
+    )
+    fig_box = px.box(x=result_df[score_column].to_list(), title="Faithfulness")
 
     return (
         {

validmind/tests/model_validation/ragas/ResponseRelevancy.py

@@ -133,8 +133,10 @@ def ResponseRelevancy(
 
     score_column = "answer_relevancy"
 
-    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
-    fig_box = px.box(x=result_df[score_column].to_list())
+    fig_histogram = px.histogram(
+        x=result_df[score_column].to_list(), nbins=10, title="Response Relevancy"
+    )
+    fig_box = px.box(x=result_df[score_column].to_list(), title="Response Relevancy")
 
     return (
         {

validmind/tests/model_validation/ragas/SemanticSimilarity.py

@@ -112,8 +112,10 @@ def SemanticSimilarity(
 
     score_column = "semantic_similarity"
 
-    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
-    fig_box = px.box(x=result_df[score_column].to_list())
+    fig_histogram = px.histogram(
+        x=result_df[score_column].to_list(), nbins=10, title="Semantic Similarity"
+    )
+    fig_box = px.box(x=result_df[score_column].to_list(), title="Semantic Similarity")
 
     return (
         {

validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.py

@@ -2,6 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Dict, List, Optional, Union
+
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
@@ -12,7 +14,12 @@ from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
-def find_optimal_threshold(y_true, y_prob, method="youden", target_recall=None):
+def find_optimal_threshold(
+    y_true: np.ndarray,
+    y_prob: np.ndarray,
+    method: str = "youden",
+    target_recall: Optional[float] = None,
+) -> Dict[str, Union[str, float]]:
     """
     Find the optimal classification threshold using various methods.
 
@@ -80,8 +87,11 @@ def find_optimal_threshold(y_true, y_prob, method="youden", target_recall=None):
 @tags("model_validation", "threshold_optimization", "classification_metrics")
 @tasks("classification")
 def ClassifierThresholdOptimization(
-    dataset: VMDataset, model: VMModel, methods=None, target_recall=None
-):
+    dataset: VMDataset,
+    model: VMModel,
+    methods: Optional[List[str]] = None,
+    target_recall: Optional[float] = None,
+) -> Dict[str, Union[pd.DataFrame, go.Figure]]:
     """
     Analyzes and visualizes different threshold optimization methods for binary classification models.
 

validmind/tests/model_validation/sklearn/OverfitDiagnosis.py

@@ -73,6 +73,7 @@ def _prepare_results(
         columns={"shape": "training records", f"{metric}": f"training {metric}"},
         inplace=True,
     )
+    results["test records"] = results_test["shape"]
     results[f"test {metric}"] = results_test[metric]
 
     # Adjust gap calculation based on metric directionality
@@ -292,7 +293,8 @@ def OverfitDiagnosis(
                 {
                     "Feature": feature_column,
                     "Slice": row["slice"],
-                    "Number of Records": row["training records"],
+                    "Number of Training Records": row["training records"],
+                    "Number of Test Records": row["test records"],
                     f"Training {metric.upper()}": row[f"training {metric}"],
                     f"Test {metric.upper()}": row[f"test {metric}"],
                     "Gap": row["gap"],

validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py

@@ -3,10 +3,12 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, List, Optional, Union
 from warnings import filters as _warnings_filters
 
 import matplotlib.pyplot as plt
 import numpy as np
+import pandas as pd
 import shap
 
 from validmind import RawData, tags, tasks
@@ -18,7 +20,10 @@ from validmind.vm_models import VMDataset, VMModel
 logger = get_logger(__name__)
 
 
-def select_shap_values(shap_values, class_of_interest):
+def select_shap_values(
+    shap_values: Union[np.ndarray, List[np.ndarray]],
+    class_of_interest: Optional[int] = None,
+) -> np.ndarray:
     """Selects SHAP values for binary or multiclass classification.
 
     For regression models, returns the SHAP values directly as there are no classes.
@@ -41,32 +46,30 @@ def select_shap_values(shap_values, class_of_interest):
     """
     if not isinstance(shap_values, list):
         # For regression, return the SHAP values as they are
-        # TODO: shap_values is always an array of all predictions, how is the if above supposed to work?
-        # logger.info("Returning SHAP values as-is.")
-        return shap_values
-
-    num_classes = len(shap_values)
-
-    # Default to class 1 for binary classification where no class is specified
-    if num_classes == 2 and class_of_interest is None:
-        logger.debug("Using SHAP values for class 1 (positive class).")
-        return shap_values[1]
+        selected_values = shap_values
+    else:
+        num_classes = len(shap_values)
+        # Default to class 1 for binary classification where no class is specified
+        if num_classes == 2 and class_of_interest is None:
+            selected_values = shap_values[1]
+        # Otherwise, use the specified class_of_interest
+        elif class_of_interest is not None and 0 <= class_of_interest < num_classes:
+            selected_values = shap_values[class_of_interest]
+        else:
+            raise ValueError(
+                f"Invalid class_of_interest: {class_of_interest}. Must be between 0 and {num_classes - 1}."
+            )
 
-    # Otherwise, use the specified class_of_interest
-    if (
-        class_of_interest is None
-        or class_of_interest < 0
-        or class_of_interest >= num_classes
-    ):
-        raise ValueError(
-            f"Invalid class_of_interest: {class_of_interest}. Must be between 0 and {num_classes - 1}."
-        )
+    # Add type conversion here to ensure proper float array
+    if hasattr(selected_values, "dtype"):
+        selected_values = np.array(selected_values, dtype=np.float64)
 
-    logger.debug(f"Using SHAP values for class {class_of_interest}.")
-    return shap_values[class_of_interest]
+    return selected_values
 
 
-def generate_shap_plot(type_, shap_values, x_test):
+def generate_shap_plot(
+    type_: str, shap_values: np.ndarray, x_test: Union[np.ndarray, pd.DataFrame]
+) -> plt.Figure:
     """Plots two types of SHAP global importance (SHAP).
 
     Args:
@@ -117,8 +120,8 @@ def SHAPGlobalImportance(
     dataset: VMDataset,
     kernel_explainer_samples: int = 10,
     tree_or_linear_explainer_samples: int = 200,
-    class_of_interest: int = None,
-):
+    class_of_interest: Optional[int] = None,
+) -> Dict[str, Union[plt.Figure, Dict[str, float]]]:
     """
     Evaluates and visualizes global feature importance using SHAP values for model explanation and risk identification.