validmind 2.8.10__py3-none-any.whl → 2.8.20__py3-none-any.whl

validmind/tests/ongoing_monitoring/CalibrationCurveDrift.py

@@ -9,7 +9,7 @@ import pandas as pd
 import plotly.graph_objects as go
 from sklearn.calibration import calibration_curve
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 
@@ -217,4 +217,14 @@ def CalibrationCurveDrift(
         fig,
         {"Mean Predicted Probabilities": pred_df, "Fraction of Positives": true_df},
         pass_fail_bool,
+        RawData(
+            prob_true_ref=prob_true_ref,
+            prob_pred_ref=prob_pred_ref,
+            prob_true_mon=prob_true_mon,
+            prob_pred_mon=prob_pred_mon,
+            bin_labels=bin_labels,
+            model=model.input_id,
+            dataset_ref=datasets[0].input_id,
+            dataset_mon=datasets[1].input_id,
+        ),
     )

validmind/tests/ongoing_monitoring/ClassificationAccuracyDrift.py

@@ -8,7 +8,7 @@ import numpy as np
 import pandas as pd
 from sklearn.metrics import classification_report
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -145,4 +145,12 @@ def ClassificationAccuracyDrift(
     # Calculate overall pass/fail
     pass_fail_bool = (df["Pass/Fail"] == "Pass").all()
 
-    return ({"Classification Accuracy Metrics": df}, pass_fail_bool)
+    raw_data = RawData(
+        report_reference=report_ref,
+        report_monitoring=report_mon,
+        model=model.input_id,
+        dataset_reference=datasets[0].input_id,
+        dataset_monitoring=datasets[1].input_id,
+    )
+
+    return ({"Classification Accuracy Metrics": df}, pass_fail_bool, raw_data)

validmind/tests/ongoing_monitoring/ConfusionMatrixDrift.py

@@ -8,7 +8,7 @@ import numpy as np
 import pandas as pd
 from sklearn.metrics import confusion_matrix
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -190,4 +190,11 @@ def ConfusionMatrixDrift(
     return (
         {"Confusion Matrix Metrics": metrics_df, "Sample Counts": counts_df},
         pass_fail_bool,
+        RawData(
+            confusion_matrix_reference=cm_ref,
+            confusion_matrix_monitoring=cm_mon,
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
     )

validmind/tests/ongoing_monitoring/CumulativePredictionProbabilitiesDrift.py

@@ -8,7 +8,7 @@ import numpy as np
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -83,6 +83,7 @@ def CumulativePredictionProbabilitiesDrift(
     diff_color = "rgba(148, 103, 189, 0.8)"  # Purple with 0.8 opacity
 
     figures = []
+    raw_data = {}
     for class_value in classes:
         # Create figure with secondary y-axis
         fig = make_subplots(
@@ -175,4 +176,19 @@ def CumulativePredictionProbabilitiesDrift(
 
         figures.append(fig)
 
-    return tuple(figures)
+        # Store raw data for current class
+        raw_data[f"class_{class_value}_ref_probs"] = ref_probs
+        raw_data[f"class_{class_value}_mon_probs"] = mon_probs
+        raw_data[f"class_{class_value}_ref_sorted"] = ref_sorted
+        raw_data[f"class_{class_value}_ref_cumsum"] = ref_cumsum
+        raw_data[f"class_{class_value}_mon_sorted"] = mon_sorted
+        raw_data[f"class_{class_value}_mon_cumsum"] = mon_cumsum
+
+    return tuple(figures) + (
+        RawData(
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+            **raw_data,
+        ),
+    )

validmind/tests/ongoing_monitoring/FeatureDrift.py

@@ -6,7 +6,7 @@ import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 def calculate_psi_score(actual, expected):
@@ -183,4 +183,11 @@ def FeatureDrift(
     # Calculate overall pass/fail
     pass_fail_bool = (psi_df["Pass/Fail"] == "Pass").all()
 
-    return ({"PSI Scores": psi_df}, *figures, pass_fail_bool)
+    # Prepare raw data
+    raw_data = RawData(
+        distributions=distributions,
+        dataset_reference=datasets[0].input_id,
+        dataset_monitoring=datasets[1].input_id,
+    )
+
+    return ({"PSI Scores": psi_df}, *figures, pass_fail_bool, raw_data)

validmind/tests/ongoing_monitoring/PredictionAcrossEachFeature.py

@@ -5,7 +5,7 @@
 
 import matplotlib.pyplot as plt
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization")
@@ -74,4 +74,10 @@ def PredictionAcrossEachFeature(datasets, model):
         figures_to_save.append(fig)
         plt.close()
 
-    return tuple(figures_to_save)
+    return tuple(figures_to_save), RawData(
+        y_prob_reference=y_prob_reference,
+        y_prob_monitoring=y_prob_monitoring,
+        model=model.input_id,
+        dataset_reference=datasets[0].input_id,
+        dataset_monitoring=datasets[1].input_id,
+    )

validmind/tests/ongoing_monitoring/PredictionCorrelation.py

@@ -5,7 +5,7 @@
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization")
@@ -140,4 +140,15 @@ def PredictionCorrelation(datasets, model, drift_pct_threshold=20):
     # Calculate overall pass/fail
     pass_fail_bool = (corr_final["Pass/Fail"] == "Pass").all()
 
-    return ({"Correlation Pair Table": corr_final}, fig, pass_fail_bool)
+    return (
+        {"Correlation Pair Table": corr_final},
+        fig,
+        pass_fail_bool,
+        RawData(
+            reference_correlations=corr_ref.to_dict(),
+            monitoring_correlations=corr_mon.to_dict(),
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
+    )

validmind/tests/ongoing_monitoring/PredictionProbabilitiesHistogramDrift.py

@@ -10,7 +10,7 @@ import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 from scipy import stats
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -201,4 +201,15 @@ def PredictionProbabilitiesHistogramDrift(
             }
         )
 
-    return fig, tables, all_passed
+    return (
+        fig,
+        tables,
+        all_passed,
+        RawData(
+            reference_probabilities=y_prob_ref,
+            monitoring_probabilities=y_prob_mon,
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
+    )

validmind/tests/ongoing_monitoring/ROCCurveDrift.py

@@ -8,7 +8,7 @@ import numpy as np
 import plotly.graph_objects as go
 from sklearn.metrics import roc_auc_score, roc_curve
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 
@@ -147,4 +147,18 @@ def ROCCurveDrift(datasets: List[VMDataset], model: VMModel):
         height=500,
     )
 
-    return fig1, fig2
+    return (
+        fig1,
+        fig2,
+        RawData(
+            fpr_ref=fpr_ref,
+            tpr_ref=tpr_ref,
+            auc_ref=auc_ref,
+            fpr_mon=fpr_mon,
+            tpr_mon=tpr_mon,
+            auc_mon=auc_mon,
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
+    )

validmind/tests/ongoing_monitoring/ScoreBandsDrift.py

@@ -7,7 +7,7 @@ from typing import List
 import numpy as np
 import pandas as pd
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -209,4 +209,13 @@ def ScoreBandsDrift(
         tables[table_name] = pd.DataFrame(rows)
         all_passed &= metric_passed
 
-    return tables, all_passed
+    # Collect raw data
+    raw_data = RawData(
+        ref_results=ref_results,
+        mon_results=mon_results,
+        model=model.input_id,
+        dataset_reference=datasets[0].input_id,
+        dataset_monitoring=datasets[1].input_id,
+    )
+
+    return tables, all_passed, raw_data

validmind/tests/ongoing_monitoring/TargetPredictionDistributionPlot.py

@@ -7,7 +7,7 @@ import plotly.figure_factory as ff
 import plotly.graph_objects as go
 from scipy.stats import kurtosis, skew
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization")
@@ -142,4 +142,15 @@ def TargetPredictionDistributionPlot(datasets, model, drift_pct_threshold=20):
 
     pass_fail_bool = (moments["Pass/Fail"] == "Pass").all()
 
-    return ({"Distribution Moments": moments}, fig, pass_fail_bool)
+    return (
+        {"Distribution Moments": moments},
+        fig,
+        pass_fail_bool,
+        RawData(
+            pred_ref=pred_ref,
+            pred_monitor=pred_monitor,
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
+    )

validmind/tests/output.py

@@ -9,6 +9,7 @@ from uuid import uuid4
 import numpy as np
 import pandas as pd
 
+from validmind.utils import is_html, md_to_html
 from validmind.vm_models.figure import (
     Figure,
     is_matplotlib_figure,
@@ -77,30 +78,72 @@ class FigureOutputHandler(OutputHandler):
 
 class TableOutputHandler(OutputHandler):
     def can_handle(self, item: Any) -> bool:
-        return isinstance(item, (list, pd.DataFrame, dict, ResultTable))
+        return isinstance(item, (list, pd.DataFrame, dict, ResultTable, tuple))
+
+    def _convert_simple_type(self, data: Any) -> pd.DataFrame:
+        """Convert a simple data type to a DataFrame."""
+        if isinstance(data, dict):
+            return pd.DataFrame([data])
+        elif data is None:
+            return pd.DataFrame()
+        else:
+            raise ValueError(f"Cannot convert {type(data)} to DataFrame")
+
+    def _convert_list(self, data_list: List) -> pd.DataFrame:
+        """Convert a list to a DataFrame."""
+        if not data_list:
+            return pd.DataFrame()
+
+        try:
+            return pd.DataFrame(data_list)
+        except Exception as e:
+            # If conversion fails, try to handle common cases
+            if all(
+                isinstance(item, (int, float, str, bool, type(None)))
+                for item in data_list
+            ):
+                return pd.DataFrame({"Values": data_list})
+            else:
+                raise ValueError(f"Could not convert list to DataFrame: {e}")
+
+    def _convert_to_dataframe(self, table_data: Any) -> pd.DataFrame:
+        """Convert various data types to a pandas DataFrame."""
+        # Handle special cases by type
+        if isinstance(table_data, pd.DataFrame):
+            return table_data
+        elif isinstance(table_data, (dict, str, type(None))):
+            return self._convert_simple_type(table_data)
+        elif isinstance(table_data, tuple):
+            return self._convert_list(list(table_data))
+        elif isinstance(table_data, list):
+            return self._convert_list(table_data)
+        else:
+            # If we reach here, we don't know how to handle this type
+            raise ValueError(
+                f"Invalid table format: must be a list of dictionaries or a DataFrame, got {type(table_data)}"
+            )
 
     def process(
         self,
-        item: Union[List[Dict[str, Any]], pd.DataFrame, Dict[str, Any], ResultTable],
+        item: Union[
+            List[Dict[str, Any]], pd.DataFrame, Dict[str, Any], ResultTable, str, tuple
+        ],
         result: TestResult,
     ) -> None:
+        # Convert to a dictionary of tables if not already
         tables = item if isinstance(item, dict) else {"": item}
 
         for table_name, table_data in tables.items():
-            # if already a ResultTable, add it directly
+            # If already a ResultTable, add it directly
             if isinstance(table_data, ResultTable):
                 result.add_table(table_data)
                 continue
 
-            if not isinstance(table_data, (list, pd.DataFrame)):
-                raise ValueError(
-                    "Invalid table format: must be a list of dictionaries or a DataFrame"
-                )
-
-            if isinstance(table_data, list):
-                table_data = pd.DataFrame(table_data)
+            # Convert the data to a DataFrame using our helper method
+            df = self._convert_to_dataframe(table_data)
 
-            result.add_table(ResultTable(data=table_data, title=table_name or None))
+            # Add the resulting DataFrame as a table to the resul
+            result.add_table(ResultTable(data=df, title=table_name or None))
 
 
 class RawDataOutputHandler(OutputHandler):
@@ -111,6 +154,17 @@ class RawDataOutputHandler(OutputHandler):
         result.raw_data = item
 
 
+class StringOutputHandler(OutputHandler):
+    def can_handle(self, item: Any) -> bool:
+        return isinstance(item, str)
+
+    def process(self, item: Any, result: TestResult) -> None:
+        if not is_html(item):
+            item = md_to_html(item, mathml=True)
+
+        result.description = item
+
+
 def process_output(item: Any, result: TestResult) -> None:
     """Process a single test output item and update the TestResult."""
     handlers = [
@@ -119,6 +173,7 @@ def process_output(item: Any, result: TestResult) -> None:
         FigureOutputHandler(),
         TableOutputHandler(),
         RawDataOutputHandler(),
+        StringOutputHandler(),
     ]
 
     for handler in handlers:

validmind/tests/prompt_validation/Clarity.py

@@ -106,5 +106,5 @@ def Clarity(model, min_threshold=7):
             }
         ],
         passed,
-        RawData(response=response),
+        RawData(response=response, model=model.input_id),
     )

validmind/tests/prompt_validation/NegativeInstruction.py

@@ -117,5 +117,5 @@ def NegativeInstruction(model, min_threshold=7):
             }
         ],
         passed,
-        RawData(model_response=response),
+        RawData(model_response=response, model=model.input_id),
     )

validmind/tests/prompt_validation/Robustness.py

@@ -130,5 +130,10 @@ def Robustness(model, dataset, num_tests=10):
     return (
         results,
         all(result["Pass/Fail"] == "Pass" for result in results),
-        RawData(generated_inputs=generated_inputs, responses=responses),
+        RawData(
+            generated_inputs=generated_inputs,
+            responses=responses,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/prompt_validation/Specificity.py

@@ -113,5 +113,5 @@ def Specificity(model, min_threshold=7):
             }
         ],
         passed,
-        RawData(response=response),
+        RawData(response=response, model=model.input_id),
     )

validmind/tests/run.py

@@ -76,7 +76,7 @@ def _get_run_metadata(**metadata: Dict[str, Any]) -> Dict[str, Any]:
 
 def _get_test_kwargs(
     test_func: callable, inputs: Dict[str, Any], params: Dict[str, Any]
-):
+) -> Tuple[Dict[str, Any], Dict[str, Any]]:
     """Insepect function signature to build kwargs to pass the inputs and params
     that the test function expects
 
@@ -93,7 +93,7 @@ def _get_test_kwargs(
         params (dict): Test parameters e.g. {"param1": 1, "param2": 2}
 
     Returns:
-        tuple: Tuple of input and param kwargs
+        Tuple[Dict[str, Any], Dict[str, Any]]: Tuple of input and param kwargs
     """
     input_kwargs = {}  # map function inputs (`dataset` etc) to actual objects
 
@@ -222,6 +222,7 @@ def _run_comparison_test(
     params: Union[Dict[str, Any], None],
     param_grid: Union[Dict[str, List[Any]], List[Dict[str, Any]], None],
     title: Optional[str] = None,
+    show_params: bool = True,
 ):
     """Run a comparison test i.e. a test that compares multiple outputs of a test across
     different input and/or param combinations"""
@@ -242,6 +243,7 @@ def _run_comparison_test(
             show=False,
             generate_description=False,
             title=title,
+            show_params=show_params,
         )
         for config in run_test_configs
     ]
@@ -253,7 +255,9 @@ def _run_comparison_test(
     else:
         test_doc = describe_test(test_id, raw=True)["Description"]
 
-    combined_outputs, combined_inputs, combined_params = combine_results(results)
+    combined_outputs, combined_inputs, combined_params = combine_results(
+        results, show_params
+    )
 
     return build_test_result(
         outputs=combined_outputs,
@@ -265,7 +269,12 @@ def _run_comparison_test(
     )
 
 
-def _run_test(test_id: TestID, inputs: Dict[str, Any], params: Dict[str, Any]):
+def _run_test(
+    test_id: TestID,
+    inputs: Dict[str, Any],
+    params: Dict[str, Any],
+    title: Optional[str] = None,
+):
     """Run a standard test and return a TestResult object"""
     test_func = load_test(test_id)
     input_kwargs, param_kwargs = _get_test_kwargs(
@@ -282,6 +291,7 @@ def _run_test(test_id: TestID, inputs: Dict[str, Any], params: Dict[str, Any]):
         test_doc=getdoc(test_func),
         inputs=input_kwargs,
         params=param_kwargs,
+        title=title,
     )
 
 
@@ -297,6 +307,7 @@ def run_test(  # noqa: C901
     generate_description: bool = True,
     title: Optional[str] = None,
     post_process_fn: Union[Callable[[TestResult], None], None] = None,
+    show_params: bool = True,
     **kwargs,
 ) -> TestResult:
     """Run a ValidMind or custom test
@@ -321,6 +332,7 @@ def run_test(  # noqa: C901
         generate_description (bool, optional): Whether to generate a description. Defaults to True.
         title (str, optional): Custom title for the test result
         post_process_fn (Callable[[TestResult], None], optional): Function to post-process the test result
+        show_params (bool, optional): Whether to include parameter values in figure titles for comparison tests. Defaults to True.
 
     Returns:
         TestResult: A TestResult object containing the test results
@@ -358,6 +370,7 @@ def run_test(  # noqa: C901
             input_grid=input_grid,
             params=params,
             param_grid=param_grid,
+            show_params=show_params,
         )
 
     elif unit_metrics:
@@ -375,7 +388,7 @@ def run_test(  # noqa: C901
         )
 
     else:
-        result = _run_test(test_id, inputs, params)
+        result = _run_test(test_id, inputs, params, title)
 
     end_time = time.perf_counter()
     result.metadata = _get_run_metadata(duration_seconds=end_time - start_time)
@@ -383,15 +396,16 @@ def run_test(  # noqa: C901
     if post_process_fn:
         result = post_process_fn(result)
 
-    result.description = get_result_description(
-        test_id=test_id,
-        test_description=result.doc,
-        tables=result.tables,
-        figures=result.figures,
-        metric=result.metric,
-        should_generate=generate_description,
-        title=title,
-    )
+    if not result.description:
+        result.description = get_result_description(
+            test_id=test_id,
+            test_description=result.doc,
+            tables=result.tables,
+            figures=result.figures,
+            metric=result.metric,
+            should_generate=generate_description,
+            title=title,
+        )
 
     if show:
         result.show()

validmind/tests/test_providers.py

@@ -7,7 +7,7 @@ import os
 import re
 import sys
 from pathlib import Path
-from typing import List, Protocol
+from typing import Any, Callable, List, Protocol
 
 from validmind.logging import get_logger
 
@@ -95,45 +95,38 @@ class LocalTestProvider:
         """
         self.root_folder = os.path.abspath(root_folder)
 
-    def list_tests(self):
+    def list_tests(self) -> List[str]:
         """List all tests in the given namespace
 
         Returns:
             list: A list of test IDs
         """
-        test_ids = []
-
+        test_files = []
         for root, _, files in os.walk(self.root_folder):
-            for filename in files:
-                if not filename.endswith(".py") or filename.startswith("__"):
-                    continue
-
-                path = Path(root) / filename
-                if not _is_test_file(path):
+            for file in files:
+                if not file.endswith(".py"):
                     continue
 
-                rel_path = path.relative_to(self.root_folder)
-
-                test_id_parts = [p.stem for p in rel_path.parents if p.stem][::-1]
-                test_id_parts.append(path.stem)
-                test_ids.append(".".join(test_id_parts))
+                path = Path(os.path.join(root, file))
+                if _is_test_file(path):
+                    rel_path = os.path.relpath(path, self.root_folder)
+                    test_id = os.path.splitext(rel_path)[0].replace(os.sep, ".")
+                    test_files.append(test_id)
 
-        return sorted(test_ids)
+        return test_files
 
-    def load_test(self, test_id: str):
-        """
-        Load the test identified by the given test_id.
+    def load_test(self, test_id: str) -> Callable[..., Any]:
+        """Load the test function identified by the given test_id
 
         Args:
-            test_id (str): The identifier of the test. This corresponds to the relative
-            path of the python file from the root folder, with slashes replaced by dots
+            test_id (str): The test ID (does not contain the namespace under which
+                the test is registered)
 
         Returns:
-            The test class that matches the last part of the test_id.
+            callable: The test function
 
         Raises:
-            LocalTestProviderLoadModuleError: If the test module cannot be imported
-            LocalTestProviderLoadTestError: If the test class cannot be found in the module
+            FileNotFoundError: If the test is not found
         """
         # Convert test_id to file path
         file_path = os.path.join(self.root_folder, f"{test_id.replace('.', '/')}.py")
@@ -162,28 +155,28 @@ class LocalTestProvider:
 
 
 class ValidMindTestProvider:
-    """Test provider for ValidMind tests"""
+    """Provider for built-in ValidMind tests"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         # two subproviders: unit_metrics and normal tests
-        self.metrics_provider = LocalTestProvider(
+        self.unit_metrics_provider = LocalTestProvider(
             os.path.join(os.path.dirname(__file__), "..", "unit_metrics")
         )
-        self.tests_provider = LocalTestProvider(os.path.dirname(__file__))
+        self.test_provider = LocalTestProvider(os.path.dirname(__file__))
 
     def list_tests(self) -> List[str]:
-        """List all tests in the ValidMind test provider"""
+        """List all tests in the given namespace"""
         metric_ids = [
-            f"unit_metrics.{test}" for test in self.metrics_provider.list_tests()
+            f"unit_metrics.{test}" for test in self.unit_metrics_provider.list_tests()
         ]
-        test_ids = self.tests_provider.list_tests()
+        test_ids = self.test_provider.list_tests()
 
         return metric_ids + test_ids
 
-    def load_test(self, test_id: str) -> callable:
-        """Load a ValidMind test or unit metric"""
+    def load_test(self, test_id: str) -> Callable[..., Any]:
+        """Load the test function identified by the given test_id"""
         return (
-            self.metrics_provider.load_test(test_id.replace("unit_metrics.", ""))
+            self.unit_metrics_provider.load_test(test_id.replace("unit_metrics.", ""))
             if test_id.startswith("unit_metrics.")
-            else self.tests_provider.load_test(test_id)
+            else self.test_provider.load_test(test_id)
         )