validmind 2.8.27__py3-none-any.whl → 2.8.29__py3-none-any.whl

validmind/tests/data_validation/nlp/Sentiment.py

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 
+from typing import Tuple
+
 import matplotlib.pyplot as plt
 import nltk
 import seaborn as sns
@@ -13,7 +15,7 @@ from validmind import RawData, tags, tasks
 
 @tags("nlp", "text_data", "data_validation")
 @tasks("nlp")
-def Sentiment(dataset):
+def Sentiment(dataset) -> Tuple[plt.Figure, RawData]:
     """
     Analyzes the sentiment of text data within a dataset using the VADER sentiment analysis tool.
 

validmind/tests/data_validation/nlp/StopWords.py

@@ -7,6 +7,7 @@ Threshold based tests
 """
 
 from collections import defaultdict
+from typing import Dict, Tuple
 
 import nltk
 import pandas as pd
@@ -21,7 +22,7 @@ from validmind.vm_models import VMDataset
 @tasks("text_classification", "text_summarization")
 def StopWords(
     dataset: VMDataset, min_percent_threshold: float = 0.5, num_words: int = 25
-):
+) -> Tuple[Dict[str, pd.DataFrame], go.Figure, bool, RawData]:
     """
     Evaluates and visualizes the frequency of English stop words in a text dataset against a defined threshold.
 

validmind/tests/data_validation/nlp/TextDescription.py

@@ -3,10 +3,12 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import string
+from typing import Tuple
 
 import nltk
 import pandas as pd
 import plotly.express as px
+import plotly.graph_objects as go
 from nltk.corpus import stopwords
 
 from validmind import RawData, tags, tasks
@@ -94,7 +96,7 @@ def TextDescription(
         "``",
     },
     lang: str = "english",
-):
+) -> Tuple[go.Figure, RawData]:
     """
     Conducts comprehensive textual analysis on a dataset using NLTK to evaluate various parameters and generate
     visualizations.

validmind/tests/data_validation/nlp/Toxicity.py

@@ -2,6 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import evaluate
 import matplotlib.pyplot as plt
 import seaborn as sns
@@ -11,7 +13,7 @@ from validmind import RawData, tags, tasks
 
 @tags("nlp", "text_data", "data_validation")
 @tasks("nlp")
-def Toxicity(dataset):
+def Toxicity(dataset) -> Tuple[plt.Figure, RawData]:
     """
     Assesses the toxicity of text data within a dataset to visualize the distribution of toxicity scores.
 

validmind/tests/load.py

@@ -7,7 +7,17 @@
 import inspect
 import json
 from pprint import pformat
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Union,
+    get_args,
+    get_origin,
+)
 from uuid import uuid4
 
 import pandas as pd
@@ -18,12 +28,31 @@ from ..html_templates.content_blocks import test_content_block_html
 from ..logging import get_logger
 from ..utils import display, format_dataframe, fuzzy_match, md_to_html, test_id_to_name
 from ..vm_models import VMDataset, VMModel
+from ..vm_models.figure import Figure
+from ..vm_models.result import ResultTable
 from .__types__ import TestID
 from ._store import test_provider_store, test_store
 
 logger = get_logger(__name__)
 
 
+try:
+    from matplotlib.figure import Figure as MatplotlibFigure
+except ImportError:
+    MatplotlibFigure = None
+
+try:
+    from plotly.graph_objects import Figure as PlotlyFigure
+except ImportError:
+    PlotlyFigure = None
+
+FIGURE_TYPES = tuple(
+    item for item in (Figure, MatplotlibFigure, PlotlyFigure) if inspect.isclass(item)
+)
+TABLE_TYPES = (pd.DataFrame, ResultTable)
+GENERIC_TABLE_TYPES = (list, dict)
+
+
 INPUT_TYPE_MAP = {
     "dataset": VMDataset,
     "datasets": List[VMDataset],
@@ -32,6 +61,45 @@ INPUT_TYPE_MAP = {
 }
 
 
+def _inspect_return_type(annotation: Any) -> Tuple[bool, bool]:
+    """
+    Inspects a return type annotation to determine if it contains a Figure or Table.
+
+    Returns a tuple (has_figure, has_table).
+    """
+    has_figure = False
+    has_table = False
+
+    origin = get_origin(annotation)
+    args = get_args(annotation)
+
+    # A Union means the return type could be one of several types.
+    # A tuple in a type hint means multiple return values.
+    # We recursively inspect the arguments of Union and tuple.
+    if origin is Union or origin is tuple:
+        for arg in args:
+            fig, table = _inspect_return_type(arg)
+            has_figure |= fig
+            has_table |= table
+        return has_figure, has_table
+
+    check_type = origin if origin is not None else annotation
+
+    if not inspect.isclass(check_type):
+        return has_figure, has_table  # Can't do issubclass on non-class like Any
+
+    if FIGURE_TYPES and issubclass(check_type, FIGURE_TYPES):
+        has_figure = True
+
+    if TABLE_TYPES and issubclass(check_type, TABLE_TYPES):
+        has_table = True
+
+    if check_type in GENERIC_TABLE_TYPES:
+        has_table = True
+
+    return has_figure, has_table
+
+
 def _inspect_signature(
     test_func: Callable[..., Any],
 ) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, Dict[str, Any]]]:
@@ -173,23 +241,29 @@ def _pretty_list_tests(
     tests: Dict[str, Callable[..., Any]], truncate: bool = True
 ) -> None:
     """Pretty print a list of tests"""
-    table = [
-        {
-            "ID": test_id,
-            "Name": test_id_to_name(test_id),
-            "Description": _test_description(
-                inspect.getdoc(test),
-                num_lines=(5 if truncate else 999999),
-            ),
-            "Required Inputs": list(test.inputs.keys()),
-            "Params": test.params,
-            "Tags": test.__tags__,
-            "Tasks": test.__tasks__,
-        }
-        for test_id, test in tests.items()
-    ]
+    rows = []
+    for test_id, test in tests.items():
+        has_figure, has_table = _inspect_return_type(
+            inspect.signature(test).return_annotation
+        )
+        rows.append(
+            {
+                "ID": test_id,
+                "Name": test_id_to_name(test_id),
+                "Description": _test_description(
+                    inspect.getdoc(test),
+                    num_lines=(5 if truncate else 999999),
+                ),
+                "Has Figure": has_figure,
+                "Has Table": has_table,
+                "Required Inputs": list(test.inputs.keys()),
+                "Params": test.params,
+                "Tags": test.__tags__,
+                "Tasks": test.__tasks__,
+            }
+        )
 
-    return format_dataframe(pd.DataFrame(table))
+    return format_dataframe(pd.DataFrame(rows))
 
 
 def list_tags() -> List[str]:

validmind/tests/model_validation/BertScore.py

@@ -2,21 +2,24 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import evaluate
 import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
 from validmind.tests.utils import validate_prediction
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("nlp", "text_data", "visualization")
 @tasks("text_classification", "text_summarization")
 def BertScore(
-    dataset,
-    model,
+    dataset: VMDataset,
+    model: VMModel,
     evaluation_model="distilbert-base-uncased",
-):
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Assesses the quality of machine-generated text using BERTScore metrics and visualizes results through histograms
     and bar charts, alongside compiling a comprehensive table of descriptive statistics.

validmind/tests/model_validation/BleuScore.py

@@ -2,17 +2,22 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import evaluate
 import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
 from validmind.tests.utils import validate_prediction
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("nlp", "text_data", "visualization")
 @tasks("text_classification", "text_summarization")
-def BleuScore(dataset, model):
+def BleuScore(
+    dataset: VMDataset, model: VMModel
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Evaluates the quality of machine-generated text using BLEU metrics and visualizes the results through histograms
     and bar charts, alongside compiling a comprehensive table of descriptive statistics for BLEU scores.

validmind/tests/model_validation/ClusterSizeDistribution.py

@@ -2,6 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import pandas as pd
 import plotly.graph_objects as go
 
@@ -11,7 +13,9 @@ from validmind.vm_models import VMDataset, VMModel
 
 @tags("sklearn", "model_performance")
 @tasks("clustering")
-def ClusterSizeDistribution(dataset: VMDataset, model: VMModel):
+def ClusterSizeDistribution(
+    dataset: VMDataset, model: VMModel
+) -> Tuple[go.Figure, RawData]:
     """
     Assesses the performance of clustering models by comparing the distribution of cluster sizes in model predictions
     with the actual data.

validmind/tests/model_validation/ContextualRecall.py

@@ -2,17 +2,22 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import nltk
 import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
 from validmind.tests.utils import validate_prediction
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("nlp", "text_data", "visualization")
 @tasks("text_classification", "text_summarization")
-def ContextualRecall(dataset, model):
+def ContextualRecall(
+    dataset: VMDataset, model: VMModel
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Evaluates a Natural Language Generation model's ability to generate contextually relevant and factually correct
     text, visualizing the results through histograms and bar charts, alongside compiling a comprehensive table of

validmind/tests/model_validation/FeaturesAUC.py

@@ -2,6 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
@@ -17,7 +19,9 @@ logger = get_logger(__name__)
 
 @tags("feature_importance", "AUC", "visualization")
 @tasks("classification")
-def FeaturesAUC(dataset: VMDataset, fontsize: int = 12, figure_height: int = 500):
+def FeaturesAUC(
+    dataset: VMDataset, fontsize: int = 12, figure_height: int = 500
+) -> Tuple[go.Figure, RawData]:
     """
     Evaluates the discriminatory power of each individual feature within a binary classification model by calculating
     the Area Under the Curve (AUC) for each feature separately.

validmind/tests/model_validation/MeteorScore.py

@@ -2,17 +2,22 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import evaluate
 import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
 from validmind.tests.utils import validate_prediction
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("nlp", "text_data", "visualization")
 @tasks("text_classification", "text_summarization")
-def MeteorScore(dataset, model):
+def MeteorScore(
+    dataset: VMDataset, model: VMModel
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Assesses the quality of machine-generated translations by comparing them to human-produced references using the
     METEOR score, which evaluates precision, recall, and word order.

validmind/tests/model_validation/ModelMetadata.py

@@ -6,11 +6,12 @@ import pandas as pd
 
 from validmind import tags, tasks
 from validmind.utils import get_model_info
+from validmind.vm_models import VMModel
 
 
 @tags("model_training", "metadata")
 @tasks("regression", "time_series_forecasting")
-def ModelMetadata(model):
+def ModelMetadata(model: VMModel) -> pd.DataFrame:
     """
     Compare metadata of different models and generate a summary table with the results.
 

validmind/tests/model_validation/ModelPredictionResiduals.py

@@ -2,18 +2,26 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Optional, Tuple
+
 import pandas as pd
 import plotly.graph_objects as go
 from scipy.stats import kstest
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("regression")
 @tasks("residual_analysis", "visualization")
 def ModelPredictionResiduals(
-    dataset, model, nbins=100, p_value_threshold=0.05, start_date=None, end_date=None
-):
+    dataset: VMDataset,
+    model: VMModel,
+    nbins: int = 100,
+    p_value_threshold: float = 0.05,
+    start_date: Optional[str] = None,
+    end_date: Optional[str] = None,
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Assesses normality and behavior of residuals in regression models through visualization and statistical tests.
 

validmind/tests/model_validation/RegardScore.py

@@ -2,17 +2,23 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import evaluate
 import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
 from validmind.tests.utils import validate_prediction
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("nlp", "text_data", "visualization")
 @tasks("text_classification", "text_summarization")
-def RegardScore(dataset, model):
+def RegardScore(
+    dataset: VMDataset,
+    model: VMModel,
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Assesses the sentiment and potential biases in text generated by NLP models by computing and visualizing regard
     scores.

validmind/tests/model_validation/RegressionResidualsPlot.py

@@ -2,6 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import numpy as np
 import plotly.figure_factory as ff
 import plotly.graph_objects as go
@@ -12,7 +14,9 @@ from validmind.vm_models import VMDataset, VMModel
 
 @tags("model_performance", "visualization")
 @tasks("regression")
-def RegressionResidualsPlot(model: VMModel, dataset: VMDataset, bin_size: float = 0.1):
+def RegressionResidualsPlot(
+    model: VMModel, dataset: VMDataset, bin_size: float = 0.1
+) -> Tuple[go.Figure, go.Figure, RawData]:
     """
     Evaluates regression model performance using residual distribution and actual vs. predicted plots.
 

validmind/tests/model_validation/RougeScore.py

@@ -2,16 +2,23 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import pandas as pd
 import plotly.graph_objects as go
 from rouge import Rouge
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("nlp", "text_data", "visualization")
 @tasks("text_classification", "text_summarization")
-def RougeScore(dataset, model, metric="rouge-1"):
+def RougeScore(
+    dataset: VMDataset,
+    model: VMModel,
+    metric: str = "rouge-1",
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Assesses the quality of machine-generated text using ROUGE metrics and visualizes the results to provide
     comprehensive performance insights.

validmind/tests/model_validation/TimeSeriesPredictionWithCI.py

@@ -2,17 +2,24 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 from scipy.stats import norm
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("model_predictions", "visualization")
 @tasks("regression", "time_series_forecasting")
-def TimeSeriesPredictionWithCI(dataset, model, confidence=0.95):
+def TimeSeriesPredictionWithCI(
+    dataset: VMDataset,
+    model: VMModel,
+    confidence: float = 0.95,
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Assesses predictive accuracy and uncertainty in time series models, highlighting breaches beyond confidence
     intervals.

validmind/tests/model_validation/TimeSeriesPredictionsPlot.py

@@ -2,14 +2,20 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("model_predictions", "visualization")
 @tasks("regression", "time_series_forecasting")
-def TimeSeriesPredictionsPlot(dataset, model):
+def TimeSeriesPredictionsPlot(
+    dataset: VMDataset,
+    model: VMModel,
+) -> Tuple[go.Figure, RawData]:
     """
     Plot actual vs predicted values for time series data and generate a visual comparison for the model.
 

validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py

@@ -2,17 +2,22 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Optional, Tuple
 
 import pandas as pd
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn import metrics
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("model_performance", "sklearn")
 @tasks("regression", "time_series_forecasting")
-def TimeSeriesR2SquareBySegments(dataset, model, segments=None):
+def TimeSeriesR2SquareBySegments(
+    dataset: VMDataset, model: VMModel, segments: Optional[int] = None
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Evaluates the R-Squared values of regression models over specified time segments in time series data to assess
     segment-wise model performance.

validmind/tests/model_validation/TokenDisparity.py

@@ -2,15 +2,20 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("nlp", "text_data", "visualization")
 @tasks("text_classification", "text_summarization")
-def TokenDisparity(dataset, model):
+def TokenDisparity(
+    dataset: VMDataset, model: VMModel
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Evaluates the token disparity between reference and generated texts, visualizing the results through histograms and
     bar charts, alongside compiling a comprehensive table of descriptive statistics for token counts.

validmind/tests/model_validation/ToxicityScore.py

@@ -2,16 +2,21 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import evaluate
 import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("nlp", "text_data", "visualization")
 @tasks("text_classification", "text_summarization")
-def ToxicityScore(dataset, model):
+def ToxicityScore(
+    dataset: VMDataset, model: VMModel
+) -> Tuple[pd.DataFrame, go.Figure, RawData]:
     """
     Assesses the toxicity levels of texts generated by NLP models to identify and mitigate harmful or offensive content.
 

validmind/tests/model_validation/embeddings/ClusterDistribution.py

@@ -2,7 +2,10 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.cluster import KMeans
 
 from validmind import RawData, tags, tasks
@@ -11,7 +14,9 @@ from validmind.vm_models import VMDataset, VMModel
 
 @tags("llm", "text_data", "embeddings", "visualization")
 @tasks("feature_extraction")
-def ClusterDistribution(model: VMModel, dataset: VMDataset, num_clusters: int = 5):
+def ClusterDistribution(
+    model: VMModel, dataset: VMDataset, num_clusters: int = 5
+) -> Tuple[go.Figure, RawData]:
     """
     Assesses the distribution of text embeddings across clusters produced by a model using KMeans clustering.
 

validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py

@@ -3,18 +3,23 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 from itertools import combinations
+from typing import List, Tuple
 
 import numpy as np
 import pandas as pd
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.metrics.pairwise import cosine_similarity
 
 from validmind import RawData, tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
 @tasks("text_qa", "text_generation", "text_summarization")
-def CosineSimilarityComparison(dataset, models):
+def CosineSimilarityComparison(
+    dataset: VMDataset, models: List[VMModel]
+) -> Tuple[go.Figure, RawData, pd.DataFrame]:
     """
     Assesses the similarity between embeddings generated by different models using Cosine Similarity, providing both
     statistical and visual insights.

validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py

@@ -2,7 +2,10 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Tuple
+
 import plotly.express as px
+import plotly.graph_objects as go
 from sklearn.metrics.pairwise import cosine_similarity
 
 from validmind import RawData, tags, tasks
@@ -11,7 +14,9 @@ from validmind.vm_models import VMDataset, VMModel
 
 @tags("llm", "text_data", "embeddings", "visualization")
 @tasks("feature_extraction")
-def CosineSimilarityDistribution(dataset: VMDataset, model: VMModel):
+def CosineSimilarityDistribution(
+    dataset: VMDataset, model: VMModel
+) -> Tuple[go.Figure, RawData]:
     """
     Assesses the similarity between predicted text embeddings from a model using a Cosine Similarity distribution
     histogram.