validmind 2.8.10__py3-none-any.whl → 2.8.20__py3-none-any.whl

validmind/tests/data_validation/nlp/TextDescription.py

@@ -173,4 +173,4 @@ def TextDescription(
             )
         )
 
-    return (*figures, RawData(metrics_dataframe=metrics_df))
+    return (*figures, RawData(metrics_dataframe=metrics_df, dataset=dataset.input_id))

validmind/tests/data_validation/nlp/Toxicity.py

@@ -73,4 +73,4 @@ def Toxicity(dataset):
 
     plt.close()
 
-    return fig, RawData(toxicity_scores=toxicity_scores)
+    return fig, RawData(toxicity_scores=toxicity_scores, dataset=dataset.input_id)

validmind/tests/decorator.py

@@ -7,6 +7,7 @@
 import inspect
 import os
 from functools import wraps
+from typing import Any, Callable, List, Optional, TypeVar, Union
 
 from validmind.logging import get_logger
 
@@ -15,8 +16,10 @@ from .load import load_test
 
 logger = get_logger(__name__)
 
+F = TypeVar("F", bound=Callable[..., Any])
 
-def _get_save_func(func, test_id):
+
+def _get_save_func(func: Callable[..., Any], test_id: str) -> Callable[..., None]:
     """Helper function to save a decorated function to a file
 
     Useful when a custom test function has been created inline in a notebook or
@@ -29,7 +32,7 @@ def _get_save_func(func, test_id):
     # remove decorator line
     source = source.split("\n", 1)[1]
 
-    def save(root_folder=".", imports=None):
+    def save(root_folder: str = ".", imports: Optional[List[str]] = None) -> None:
         parts = test_id.split(".")
 
         if len(parts) > 1:
@@ -84,7 +87,7 @@ def _get_save_func(func, test_id):
     return save
 
 
-def test(func_or_id):
+def test(func_or_id: Union[Callable[..., Any], str, None]) -> Callable[[F], F]:
     """Decorator for creating and registering custom tests
 
     This decorator registers the function it wraps as a test function within ValidMind
@@ -109,14 +112,14 @@ def test(func_or_id):
     as the metric's description.
 
     Args:
-        func: The function to decorate
-        test_id: The identifier for the metric. If not provided, the function name is used.
+        func_or_id (Union[Callable[..., Any], str, None]): Either the function to decorate
+            or the test ID. If None, the function name is used.
 
     Returns:
-        The decorated function.
+        Callable[[F], F]: The decorated function.
     """
 
-    def decorator(func):
+    def decorator(func: F) -> F:
         test_id = func_or_id or f"validmind.custom_metrics.{func.__name__}"
         test_func = load_test(test_id, func, reload=True)
         test_store.register_test(test_id, test_func)
@@ -136,28 +139,28 @@ def test(func_or_id):
     return decorator
 
 
-def tasks(*tasks):
+def tasks(*tasks: str) -> Callable[[F], F]:
     """Decorator for specifying the task types that a test is designed for.
 
     Args:
         *tasks: The task types that the test is designed for.
     """
 
-    def decorator(func):
+    def decorator(func: F) -> F:
         func.__tasks__ = list(tasks)
         return func
 
     return decorator
 
 
-def tags(*tags):
+def tags(*tags: str) -> Callable[[F], F]:
     """Decorator for specifying tags for a test.
 
     Args:
         *tags: The tags to apply to the test.
     """
 
-    def decorator(func):
+    def decorator(func: F) -> F:
         func.__tags__ = list(tags)
         return func
 

validmind/tests/load.py

@@ -7,7 +7,7 @@
 import inspect
 import json
 from pprint import pformat
-from typing import List
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 from uuid import uuid4
 
 import pandas as pd
@@ -32,7 +32,10 @@ INPUT_TYPE_MAP = {
 }
 
 
-def _inspect_signature(test_func: callable):
+def _inspect_signature(
+    test_func: Callable[..., Any],
+) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, Dict[str, Any]]]:
+    """Inspect a test function's signature to get inputs and parameters"""
     inputs = {}
     params = {}
 
@@ -56,7 +59,9 @@ def _inspect_signature(test_func: callable):
     return inputs, params
 
 
-def load_test(test_id: str, test_func: callable = None, reload: bool = False):
+def load_test(
+    test_id: str, test_func: Optional[Callable[..., Any]] = None, reload: bool = False
+) -> Callable[..., Any]:
     """Load a test by test ID
 
     Test IDs are in the format `namespace.path_to_module.TestClassOrFuncName[:tag]`.
@@ -67,6 +72,8 @@ def load_test(test_id: str, test_func: callable = None, reload: bool = False):
         test_id (str): The test ID in the format `namespace.path_to_module.TestName[:tag]`
         test_func (callable, optional): The test function to load. If not provided, the
             test will be loaded from the test provider. Defaults to None.
+        reload (bool, optional): If True, reload the test even if it's already loaded.
+            Defaults to False.
     """
     # remove tag if present
     test_id = test_id.split(":", 1)[0]
@@ -109,7 +116,8 @@ def load_test(test_id: str, test_func: callable = None, reload: bool = False):
     return test_store.get_test(test_id)
 
 
-def _list_test_ids():
+def _list_test_ids() -> List[str]:
+    """List all available test IDs"""
     test_ids = []
 
     for namespace, test_provider in test_provider_store.test_providers.items():
@@ -120,7 +128,7 @@ def _list_test_ids():
     return test_ids
 
 
-def _load_tests(test_ids):
+def _load_tests(test_ids: List[str]) -> Dict[str, Callable[..., Any]]:
     """Load a set of tests, handling missing dependencies."""
     tests = {}
 
@@ -138,12 +146,12 @@ def _load_tests(test_ids):
             logger.debug(str(e))
 
             if e.extra:
-                logger.info(
+                logger.debug(
                     f"Skipping `{test_id}` as it requires extra dependencies: {e.required_dependencies}."
                     f" Please run `pip install validmind[{e.extra}]` to view and run this test."
                 )
             else:
-                logger.info(
+                logger.debug(
                     f"Skipping `{test_id}` as it requires missing dependencies: {e.required_dependencies}."
                     " Please install the missing dependencies to view and run this test."
                 )
@@ -151,7 +159,8 @@ def _load_tests(test_ids):
     return tests
 
 
-def _test_description(test_description: str, num_lines: int = 5):
+def _test_description(test_description: str, num_lines: int = 5) -> str:
+    """Format a test description"""
     description = test_description.strip("\n").strip()
 
     if len(description.split("\n")) > num_lines:
@@ -160,7 +169,10 @@ def _test_description(test_description: str, num_lines: int = 5):
     return description
 
 
-def _pretty_list_tests(tests, truncate=True):
+def _pretty_list_tests(
+    tests: Dict[str, Callable[..., Any]], truncate: bool = True
+) -> None:
+    """Pretty print a list of tests"""
     table = [
         {
             "ID": test_id,
@@ -171,6 +183,8 @@ def _pretty_list_tests(tests, truncate=True):
             ),
             "Required Inputs": list(test.inputs.keys()),
             "Params": test.params,
+            "Tags": test.__tags__,
+            "Tasks": test.__tasks__,
         }
         for test_id, test in tests.items()
     ]
@@ -178,10 +192,8 @@ def _pretty_list_tests(tests, truncate=True):
     return format_dataframe(pd.DataFrame(table))
 
 
-def list_tags():
-    """
-    List unique tags from all test classes.
-    """
+def list_tags() -> List[str]:
+    """List all unique available tags"""
 
     unique_tags = set()
 
@@ -191,7 +203,7 @@ def list_tags():
     return list(unique_tags)
 
 
-def list_tasks_and_tags(as_json=False):
+def list_tasks_and_tags(as_json: bool = False) -> Union[str, Dict[str, List[str]]]:
     """
     List all task types and their associated tags, with one row per task type and
     all tags for a task type in one row.
@@ -218,11 +230,8 @@ def list_tasks_and_tags(as_json=False):
     )
 
 
-def list_tasks():
-    """
-    List unique tasks from all test classes.
-    """
-
+def list_tasks() -> List[str]:
+    """List all unique available tasks"""
     unique_tasks = set()
 
     for test in _load_tests(list_tests(pretty=False)).values():
@@ -231,7 +240,13 @@ def list_tasks():
     return list(unique_tasks)
 
 
-def list_tests(filter=None, task=None, tags=None, pretty=True, truncate=True):
+def list_tests(
+    filter: Optional[str] = None,
+    task: Optional[str] = None,
+    tags: Optional[List[str]] = None,
+    pretty: bool = True,
+    truncate: bool = True,
+) -> Union[List[str], None]:
     """List all tests in the tests directory.
 
     Args:
@@ -245,9 +260,6 @@ def list_tests(filter=None, task=None, tags=None, pretty=True, truncate=True):
             formatted table. Defaults to True.
         truncate (bool, optional): If True, truncates the test description to the first
             line. Defaults to True. (only used if pretty=True)
-
-    Returns:
-        list or pandas.DataFrame: A list of all tests or a formatted table.
     """
     test_ids = _list_test_ids()
 
@@ -286,7 +298,9 @@ def list_tests(filter=None, task=None, tags=None, pretty=True, truncate=True):
     return _pretty_list_tests(tests, truncate=truncate)
 
 
-def describe_test(test_id: TestID = None, raw: bool = False, show: bool = True):
+def describe_test(
+    test_id: Optional[TestID] = None, raw: bool = False, show: bool = True
+) -> Union[str, HTML, Dict[str, Any]]:
     """Get or show details about the test
 
     This function can be used to see test details including the test name, description,

validmind/tests/model_validation/BertScore.py

@@ -131,4 +131,10 @@ def BertScore(
     # Create a DataFrame from all collected statistics
     result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
 
-    return (result_df, *figures, RawData(bert_scores_df=metrics_df))
+    return (
+        result_df,
+        *figures,
+        RawData(
+            bert_scores_df=metrics_df, model=model.input_id, dataset=dataset.input_id
+        ),
+    )

validmind/tests/model_validation/BleuScore.py

@@ -114,4 +114,10 @@ def BleuScore(dataset, model):
     # Create a DataFrame from all collected statistics
     result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
 
-    return (result_df, *figures, RawData(bleu_scores_df=metrics_df))
+    return (
+        result_df,
+        *figures,
+        RawData(
+            bleu_scores_df=metrics_df, model=model.input_id, dataset=dataset.input_id
+        ),
+    )

validmind/tests/model_validation/ClusterSizeDistribution.py

@@ -72,4 +72,6 @@ def ClusterSizeDistribution(dataset: VMDataset, model: VMModel):
     fig.update_yaxes(title_text="Counts", showgrid=False)
     fig.update_layout(title_text="Cluster distribution", title_x=0.5, barmode="group")
 
-    return fig, RawData(cluster_counts=df_counts)
+    return fig, RawData(
+        cluster_counts=df_counts, model=model.input_id, dataset=dataset.input_id
+    )

validmind/tests/model_validation/ContextualRecall.py

@@ -118,4 +118,12 @@ def ContextualRecall(dataset, model):
     # Create a DataFrame from all collected statistics
     result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
 
-    return (result_df, *figures, RawData(contextual_recall_scores=metrics_df))
+    return (
+        result_df,
+        *figures,
+        RawData(
+            contextual_recall_scores=metrics_df,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
+    )

validmind/tests/model_validation/FeaturesAUC.py

@@ -95,4 +95,4 @@ def FeaturesAUC(dataset: VMDataset, fontsize: int = 12, figure_height: int = 500
         height=figure_height,
     )
 
-    return fig, RawData(feature_aucs=aucs)
+    return fig, RawData(feature_aucs=aucs, dataset=dataset.input_id)

validmind/tests/model_validation/MeteorScore.py

@@ -117,4 +117,10 @@ def MeteorScore(dataset, model):
     # Create a DataFrame from all collected statistics
     result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
 
-    return (result_df, *figures, RawData(meteor_scores=metrics_df))
+    return (
+        result_df,
+        *figures,
+        RawData(
+            meteor_scores=metrics_df, model=model.input_id, dataset=dataset.input_id
+        ),
+    )

validmind/tests/model_validation/ModelPredictionResiduals.py

@@ -102,4 +102,8 @@ def ModelPredictionResiduals(
     # Create a summary DataFrame for the KS normality test results
     summary_df = pd.DataFrame([summary])
 
-    return (summary_df, *figures, RawData(residuals=residuals))
+    return (
+        summary_df,
+        *figures,
+        RawData(residuals=residuals, model=model.input_id, dataset=dataset.input_id),
+    )

validmind/tests/model_validation/RegardScore.py

@@ -145,5 +145,10 @@ def RegardScore(dataset, model):
     return (
         result_df,
         *figures,
-        RawData(true_regard=true_df, pred_regard=pred_df),
+        RawData(
+            true_regard=true_df,
+            pred_regard=pred_df,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/model_validation/RegressionResidualsPlot.py

@@ -105,4 +105,13 @@ def RegressionResidualsPlot(model: VMModel, dataset: VMDataset, bin_size: float
         )
     )
 
-    return (*figures, RawData(residuals=residuals, y_true=y_true, y_pred=y_pred))
+    return (
+        *figures,
+        RawData(
+            residuals=residuals,
+            y_true=y_true,
+            y_pred=y_pred,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
+    )

validmind/tests/model_validation/RougeScore.py

@@ -121,5 +121,7 @@ def RougeScore(dataset, model, metric="rouge-1"):
     return (
         pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"}),
         *figures,
-        RawData(rouge_scores_df=df_scores),
+        RawData(
+            rouge_scores_df=df_scores, model=model.input_id, dataset=dataset.input_id
+        ),
     )

validmind/tests/model_validation/TimeSeriesPredictionWithCI.py

@@ -152,5 +152,7 @@ def TimeSeriesPredictionWithCI(dataset, model, confidence=0.95):
             z_score=z_score,
             lower_confidence=lower_conf,
             upper_confidence=upper_conf,
+            model=model.input_id,
+            dataset=dataset.input_id,
         ),
     )

validmind/tests/model_validation/TimeSeriesPredictionsPlot.py

@@ -4,7 +4,7 @@
 
 import plotly.graph_objects as go
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("model_predictions", "visualization")
@@ -70,4 +70,12 @@ def TimeSeriesPredictionsPlot(dataset, model):
         template="plotly_white",
     )
 
-    return fig
+    raw_data = RawData(
+        time_index=time_index,
+        actual_values=dataset.y,
+        predicted_values=y_pred,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )
+
+    return fig, raw_data

validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py

@@ -7,7 +7,7 @@ import pandas as pd
 import plotly.express as px
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("model_performance", "sklearn")
@@ -105,4 +105,8 @@ def TimeSeriesR2SquareBySegments(dataset, model, segments=None):
         },
     )
 
-    return fig, results_df
+    return (
+        fig,
+        results_df,
+        RawData(summary=results_df, model=model.input_id, dataset=dataset.input_id),
+    )

validmind/tests/model_validation/TokenDisparity.py

@@ -108,4 +108,8 @@ def TokenDisparity(dataset, model):
     # Create a DataFrame from all collected statistics
     result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
 
-    return (result_df, *figures, RawData(token_counts_df=df))
+    return (
+        result_df,
+        *figures,
+        RawData(token_counts_df=df, model=model.input_id, dataset=dataset.input_id),
+    )

validmind/tests/model_validation/ToxicityScore.py

@@ -146,5 +146,7 @@ def ToxicityScore(dataset, model):
             input_toxicity_df=input_df,
             true_toxicity_df=true_df,
             pred_toxicity_df=pred_df,
+            model=model.input_id,
+            dataset=dataset.input_id,
         ),
     )

validmind/tests/model_validation/embeddings/ClusterDistribution.py

@@ -62,4 +62,4 @@ def ClusterDistribution(model: VMModel, dataset: VMDataset, num_clusters: int =
         title="Embeddings Cluster Distribution",
     )
 
-    return fig, RawData(labels=labels)
+    return fig, RawData(labels=labels, model=model.input_id, dataset=dataset.input_id)

validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py

@@ -113,5 +113,9 @@ def CosineSimilarityComparison(dataset, models):
     return (
         *figures,
         stats_df,
-        RawData(similarity_matrices=pd.DataFrame(similarity_matrices)),
+        RawData(
+            similarity_matrices=pd.DataFrame(similarity_matrices),
+            dataset=dataset.input_id,
+            models=[model.input_id for model in models],
+        ),
     )

validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py

@@ -59,4 +59,8 @@ def CosineSimilarityDistribution(dataset: VMDataset, model: VMModel):
         nbins=100,
         title="Cosine Similarity Distribution",
         labels={"x": "Cosine Similarity"},
-    ), RawData(similarity_scores=similarity_scores)
+    ), RawData(
+        similarity_scores=similarity_scores,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/embeddings/CosineSimilarityHeatmap.py

@@ -81,4 +81,8 @@ def CosineSimilarityHeatmap(
         yaxis_title=yaxis_title,
     )
 
-    return fig, RawData(similarity_matrix=similarity_matrix)
+    return fig, RawData(
+        similarity_matrix=similarity_matrix,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py

@@ -77,5 +77,7 @@ def DescriptiveAnalytics(dataset: VMDataset, model: VMModel):
             embedding_means=embedding_means,
             embedding_medians=embedding_medians,
             embedding_stds=embedding_stds,
+            model=model.input_id,
+            dataset=dataset.input_id,
         ),
     )

validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py

@@ -89,4 +89,8 @@ def EmbeddingsVisualization2D(
     fig = px.scatter(**scatter_kwargs)
     fig.update_layout(width=500, height=500)
 
-    return fig, RawData(tsne_embeddings=reduced_embeddings)
+    return fig, RawData(
+        tsne_embeddings=reduced_embeddings,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py

@@ -57,7 +57,7 @@ def EuclideanDistanceComparison(dataset, models):
     figures = []
     all_stats = []
 
-    distance_matrices = {}
+    distance_matrices = []
 
     # Generate all pairs of models for comparison
     for model_A, model_B in combinations(models, 2):
@@ -105,6 +105,10 @@ def EuclideanDistanceComparison(dataset, models):
     stats_df = pd.DataFrame(all_stats)
 
     # Add raw data to return
-    raw_data = RawData(distance_matrices=pd.DataFrame(distance_matrices))
+    raw_data = RawData(
+        distance_matrices=pd.DataFrame(distance_matrices),
+        dataset=dataset.input_id,
+        models=[model.input_id for model in models],
+    )
 
     return (stats_df, *figures, raw_data)

validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py

@@ -79,4 +79,6 @@ def EuclideanDistanceHeatmap(
         yaxis_title=yaxis_title,
     )
 
-    return fig, RawData(distance_matrix=distance_matrix)
+    return fig, RawData(
+        distance_matrix=distance_matrix, model=model.input_id, dataset=dataset.input_id
+    )

validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py

@@ -90,4 +90,7 @@ def PCAComponentsPairwisePlots(dataset, model, n_components=3):
         )
         figures.append(fig)
 
-    return (*figures, RawData(pca_results=pca_df))
+    return (
+        *figures,
+        RawData(pca_results=pca_df, model=model.input_id, dataset=dataset.input_id),
+    )

validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py

@@ -97,4 +97,8 @@ def StabilityAnalysisKeyword(
         mean_similarity_threshold,
     )
 
-    return results, RawData(original_perturbed_similarity=raw_data)
+    return results, RawData(
+        original_perturbed_similarity=raw_data,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py

@@ -151,4 +151,8 @@ def StabilityAnalysisRandomNoise(
         mean_similarity_threshold,
     )
 
-    return *result, RawData(original_perturbed_similarity=raw_data)
+    return *result, RawData(
+        original_perturbed_similarity=raw_data,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py

@@ -107,4 +107,8 @@ def StabilityAnalysisSynonyms(
         mean_similarity_threshold,
     )
 
-    return *result, RawData(original_perturbed_similarity=raw_data)
+    return *result, RawData(
+        original_perturbed_similarity=raw_data,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py

@@ -134,4 +134,8 @@ def StabilityAnalysisTranslation(
         mean_similarity_threshold,
     )
 
-    return *result, RawData(original_perturbed_similarity=raw_data)
+    return *result, RawData(
+        original_perturbed_similarity=raw_data,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py

@@ -110,5 +110,10 @@ def TSNEComponentsPairwisePlots(
 
     return (
         *figures,
-        RawData(embeddings_scaled=embeddings_scaled, tsne_results=tsne_results),
+        RawData(
+            embeddings_scaled=embeddings_scaled,
+            tsne_results=tsne_results,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -123,8 +123,10 @@ def AnswerCorrectness(
 
     score_column = "answer_correctness"
 
-    fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
-    fig_box = px.box(x=result_df[score_column].to_list())
+    fig_histogram = px.histogram(
+        x=result_df[score_column].to_list(), nbins=10, title="Answer Correctness"
+    )
+    fig_box = px.box(x=result_df[score_column].to_list(), title="Answer Correctness")
 
     return (
         {
@@ -144,5 +146,5 @@ def AnswerCorrectness(
         },
         fig_histogram,
         fig_box,
-        RawData(evaluation_results=result_df),
+        RawData(evaluation_results=result_df, dataset=dataset.input_id),
     )

validmind/tests/model_validation/ragas/AspectCritic.py

@@ -195,5 +195,8 @@ def AspectCritic(
             ]
         },
         fig,
-        RawData(evaluation_results=result_df),
+        RawData(
+            evaluation_results=result_df,
+            dataset=dataset.input_id,
+        ),
     )