validmind 2.5.8__py3-none-any.whl → 2.5.15__py3-none-any.whl

validmind/tests/decorator.py

@@ -9,6 +9,7 @@
 
 import inspect
 import os
+from typing import Any, Dict, List, Tuple, Union
 from uuid import uuid4
 
 import pandas as pd
@@ -22,6 +23,8 @@ from validmind.vm_models import (
     ResultSummary,
     ResultTable,
     ResultTableMetadata,
+    VMDataset,
+    VMModel,
 )
 from validmind.vm_models.figure import (
     Figure,
@@ -36,30 +39,42 @@ from ._store import test_store
 logger = get_logger(__name__)
 
 
-def _inspect_signature(test_func: callable):
-    input_keys = ["dataset", "datasets", "model", "models"]
+_input_type_map = {
+    "dataset": VMDataset,
+    "datasets": List[VMDataset],
+    "model": VMModel,
+    "models": List[VMModel],
+}
+
 
+def _inspect_signature(test_func: callable):
     inputs = {}
     params = {}
 
     for name, arg in inspect.signature(test_func).parameters.items():
-        if name in input_keys:
-            target_dict = inputs
+        if name in _input_type_map:
+            inputs[name] = {
+                "type": _input_type_map[name],
+            }
         else:
-            target_dict = params
-
-        target_dict[name] = {
-            "type": arg.annotation,
-            "default": (
-                arg.default if arg.default is not inspect.Parameter.empty else None
-            ),
-        }
+            params[name] = {
+                "type": arg.annotation,
+                "default": (
+                    arg.default if arg.default is not inspect.Parameter.empty else None
+                ),
+            }
 
     return inputs, params
 
 
 def _build_result(  # noqa: C901
-    results, test_id, description, output_template, inputs, generate_description=True
+    results: Union[Any, Tuple[Any, ...]],
+    test_id: str,
+    inputs: List[str],
+    params: Dict[str, Any],
+    description: str = None,
+    output_template: str = None,
+    generate_description: bool = True,
 ):
     ref_id = str(uuid4())
     figure_metadata = {
@@ -70,14 +85,17 @@ def _build_result(  # noqa: C901
 
     tables = []
     figures = []
+    scalars = []
 
-    def process_item(item):
+    def process_result_item(item):
         # TOOD: build out a more robust/extensible system for this
         # TODO: custom type handlers would be really cool
 
-        # unit metrics (scalar values) - show in a simple table for now
-        if isinstance(item, int) or isinstance(item, float) or isinstance(item, str):
-            tables.append(ResultTable(data=[{test_id.split(".")[-1]: item}]))
+        # unit metrics (scalar values) - for now only one per test
+        if isinstance(item, int) or isinstance(item, float):
+            if scalars:
+                raise ValueError("Only one unit metric may be returned per test.")
+            scalars.append(item)
 
         # plots
         elif isinstance(item, Figure):
@@ -114,46 +132,66 @@ def _build_result(  # noqa: C901
     # if the results are a tuple, process each item as a separate result
     if isinstance(results, tuple):
         for item in results:
-            process_item(item)
+            process_result_item(item)
     else:
-        process_item(results)
+        process_result_item(results)
 
-    result_summary = ResultSummary(results=tables)
+    metric_inputs = [
+        sub_i.input_id if hasattr(sub_i, "input_id") else sub_i
+        for i in inputs
+        for sub_i in (i if isinstance(i, list) else [i])
+    ]
 
     return MetricResultWrapper(
         result_id=test_id,
-        metric=MetricResult(
-            key=test_id,
-            ref_id=ref_id,
-            value="Empty",
-            summary=result_summary,
+        scalar=scalars[0] if scalars else None,
+        metric=(
+            MetricResult(
+                key=test_id,
+                ref_id=ref_id,
+                value="Empty",
+                summary=ResultSummary(results=tables),
+            )
+            if tables or figures  # if tables or figures than its a traditional metric
+            else None
         ),
         figures=figures,
-        result_metadata=[
-            get_description_metadata(
-                test_id=test_id,
-                default_description=description,
-                summary=result_summary.serialize(),
-                figures=figures,
-                should_generate=generate_description,
-            )
-        ],
-        inputs=inputs,
+        result_metadata=(
+            [
+                get_description_metadata(
+                    test_id=test_id,
+                    default_description=description,
+                    summary=ResultSummary(results=tables).serialize(),
+                    figures=figures,
+                    should_generate=generate_description,
+                )
+            ]
+            if tables or figures
+            else None
+        ),
+        inputs=metric_inputs,
+        params=params,
         output_template=output_template,
     )
 
 
-def _get_run_method(func, inputs, params):
+def _get_run_method(func, func_inputs, func_params):
     def run(self: Metric):
-        input_kwargs = {}
-        for k in inputs.keys():
+        input_kwargs = {}  # map function inputs (`dataset` etc) to actual objects
+        input_ids = []  # store input_ids used so they can be logged
+        for key in func_inputs.keys():
             try:
-                input_kwargs[k] = getattr(self.inputs, k)
+                input_kwargs[key] = getattr(self.inputs, key)
+                if isinstance(input_kwargs[key], list):
+                    input_ids.extend([i.input_id for i in input_kwargs[key]])
+                else:
+                    input_ids.append(input_kwargs[key].input_id)
             except AttributeError:
-                raise MissingRequiredTestInputError(f"Missing required input: {k}.")
+                raise MissingRequiredTestInputError(f"Missing required input: {key}.")
 
         param_kwargs = {
-            k: self.params.get(k, params[k]["default"]) for k in params.keys()
+            key: self.params.get(key, func_params[key]["default"])
+            for key in func_params.keys()
         }
 
         raw_results = func(**input_kwargs, **param_kwargs)
@@ -162,8 +200,9 @@ def _get_run_method(func, inputs, params):
             results=raw_results,
             test_id=self.test_id,
             description=inspect.getdoc(self),
+            inputs=input_ids,
+            params=param_kwargs,
             output_template=self.output_template,
-            inputs=self.get_accessed_inputs(),
             generate_description=self.generate_description,
         )
 

validmind/tests/model_validation/BertScore.py

@@ -13,39 +13,48 @@ from validmind import tags, tasks
 @tasks("text_classification", "text_summarization")
 def BertScore(dataset, model):
     """
-    Evaluates the quality of machine-generated text using BERTScore metrics and visualizes the results through histograms
-    and bar charts, alongside compiling a comprehensive table of descriptive statistics for each BERTScore metric.
-
-    **Purpose:**
-    This function is designed to assess the quality of text generated by machine learning models using BERTScore metrics.
-    BERTScore evaluates text generation models' performance by calculating precision, recall, and F1 score based on BERT
-    contextual embeddings.
-
-    **Test Mechanism:**
-    The function starts by extracting the true and predicted values from the provided dataset and model. It then initializes
-    the BERTScore evaluator. For each pair of true and predicted texts, the function calculates the BERTScore metrics and
-    compiles them into a dataframe. Histograms and bar charts are generated for each BERTScore metric (Precision, Recall,
-    and F1 Score) to visualize their distribution. Additionally, a table of descriptive statistics (mean, median, standard
-    deviation, minimum, and maximum) is compiled for each metric, providing a comprehensive summary of the model's performance.
-
-    **Signs of High Risk:**
-    - Consistently low scores across BERTScore metrics could indicate poor quality in the generated text, suggesting that the model
-      fails to capture the essential content of the reference texts.
+    Assesses the quality of machine-generated text using BERTScore metrics and visualizes results through histograms
+    and bar charts, alongside compiling a comprehensive table of descriptive statistics.
+
+    ### Purpose
+
+    This function is designed to assess the quality of text generated by machine learning models using BERTScore
+    metrics. BERTScore evaluates text generation models' performance by calculating precision, recall, and F1 score
+    based on BERT contextual embeddings.
+
+    ### Test Mechanism
+
+    The function starts by extracting the true and predicted values from the provided dataset and model. It then
+    initializes the BERTScore evaluator. For each pair of true and predicted texts, the function calculates the
+    BERTScore metrics and compiles them into a dataframe. Histograms and bar charts are generated for each BERTScore
+    metric (Precision, Recall, and F1 Score) to visualize their distribution. Additionally, a table of descriptive
+    statistics (mean, median, standard deviation, minimum, and maximum) is compiled for each metric, providing a
+    comprehensive summary of the model's performance.
+
+    ### Signs of High Risk
+
+    - Consistently low scores across BERTScore metrics could indicate poor quality in the generated text, suggesting
+    that the model fails to capture the essential content of the reference texts.
     - Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
     - Low recall scores may indicate that important information from the reference text is being omitted.
-    - An imbalanced performance between precision and recall, reflected by a low F1 Score, could signal issues in the model's ability
-      to balance informativeness and conciseness.
+    - An imbalanced performance between precision and recall, reflected by a low F1 Score, could signal issues in the
+    model's ability to balance informativeness and conciseness.
 
-    **Strengths:**
-    - Provides a multifaceted evaluation of text quality through different BERTScore metrics, offering a detailed view of model performance.
-    - Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the scores.
+    ### Strengths
+
+    - Provides a multifaceted evaluation of text quality through different BERTScore metrics, offering a detailed view
+    of model performance.
+    - Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the
+    scores.
     - Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
 
-    **Limitations:**
-    - BERTScore relies on the contextual embeddings from BERT models, which may not fully capture all nuances of text similarity.
+    ### Limitations
+
+    - BERTScore relies on the contextual embeddings from BERT models, which may not fully capture all nuances of text
+    similarity.
     - The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
-    - While useful for comparison, BERTScore metrics alone do not provide a complete assessment of a model's performance and should be
-      supplemented with other metrics and qualitative analysis.
+    - While useful for comparison, BERTScore metrics alone do not provide a complete assessment of a model's
+    performance and should be supplemented with other metrics and qualitative analysis.
     """
 
     # Extract true and predicted values

validmind/tests/model_validation/BleuScore.py

@@ -16,39 +16,45 @@ def BleuScore(dataset, model):
     Evaluates the quality of machine-generated text using BLEU metrics and visualizes the results through histograms
     and bar charts, alongside compiling a comprehensive table of descriptive statistics for BLEU scores.
 
-    **Purpose:**
+    ### Purpose
+
     This function is designed to assess the quality of text generated by machine learning models using the BLEU metric.
     BLEU, which stands for Bilingual Evaluation Understudy, is a metric used to evaluate the overlap of n-grams between
     the machine-generated text and reference texts. This evaluation is crucial for tasks such as text summarization,
     machine translation, and text generation, where the goal is to produce text that accurately reflects the content
     and meaning of human-crafted references.
 
-    **Test Mechanism:**
-    The function starts by extracting the true and predicted values from the provided dataset and model. It then initializes
-    the BLEU evaluator. For each pair of true and predicted texts, the function calculates the BLEU scores and compiles them
-    into a dataframe. Histograms and bar charts are generated for the BLEU scores to visualize their distribution. Additionally,
-    a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum) is compiled for the BLEU scores,
-    providing a comprehensive summary of the model's performance.
+    ### Test Mechanism
+
+    The function starts by extracting the true and predicted values from the provided dataset and model. It then
+    initializes the BLEU evaluator. For each pair of true and predicted texts, the function calculates the BLEU scores
+    and compiles them into a dataframe. Histograms and bar charts are generated for the BLEU scores to visualize their
+    distribution. Additionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and
+    maximum) is compiled for the BLEU scores, providing a comprehensive summary of the model's performance.
+
+    ### Signs of High Risk
 
-    **Signs of High Risk:**
-    - Consistently low BLEU scores could indicate poor quality in the generated text, suggesting that the model fails to capture
-      the essential content of the reference texts.
+    - Consistently low BLEU scores could indicate poor quality in the generated text, suggesting that the model fails
+    to capture the essential content of the reference texts.
     - Low precision scores might suggest that the generated text contains a lot of redundant or irrelevant information.
     - Low recall scores may indicate that important information from the reference text is being omitted.
-    - An imbalanced performance between precision and recall, reflected by a low BLEU score, could signal issues in the model's
-      ability to balance informativeness and conciseness.
+    - An imbalanced performance between precision and recall, reflected by a low BLEU score, could signal issues in the
+    model's ability to balance informativeness and conciseness.
+
+    ### Strengths
 
-    **Strengths:**
     - Provides a straightforward and widely-used evaluation of text quality through BLEU scores.
-    - Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the scores.
+    - Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of the
+    scores.
     - Descriptive statistics offer a concise summary of the model's strengths and weaknesses in generating text.
 
-    **Limitations:**
-    - BLEU metrics primarily focus on n-gram overlap and may not fully capture semantic coherence, fluency, or grammatical quality
-      of the text.
+    ### Limitations
+
+    - BLEU metrics primarily focus on n-gram overlap and may not fully capture semantic coherence, fluency, or
+    grammatical quality of the text.
     - The evaluation relies on the availability of high-quality reference texts, which may not always be obtainable.
-    - While useful for comparison, BLEU scores alone do not provide a complete assessment of a model's performance and should be
-      supplemented with other metrics and qualitative analysis.
+    - While useful for comparison, BLEU scores alone do not provide a complete assessment of a model's performance and
+    should be supplemented with other metrics and qualitative analysis.
     """
 
     # Extract true and predicted values

validmind/tests/model_validation/ClusterSizeDistribution.py

@@ -13,40 +13,44 @@ from validmind.vm_models import Figure, Metric
 @dataclass
 class ClusterSizeDistribution(Metric):
     """
-    Compares and visualizes the distribution of cluster sizes in model predictions and actual data for assessing
-    clustering model performance.
-
-    **Purpose:** The purpose of the `ClusterSizeDistribution` metric is to assess the performance of clustering models.
-    It does this by comparing the distribution of cluster sizes in the predictions made by the model and the actual
-    data. Observing the cluster distribution helps gain insights into whether the model's output aligns well with the
-    actual dataset distribution.
-
-    **Test Mechanism:** The testing mechanism for `ClusterSizeDistribution` involves first running the clustering model
-    on the training dataset, storing predictions, and comparing these predictions with the actual output. The actual
-    and predicted outputs are then converted into pandas dataframes, which conveniently enables the use of pandas
-    built-in functions to derive cluster size distributions. Two histograms are constructed from this data: one for the
-    actual distribution and one for the predicted distribution. These histograms are then plotted side-by-side for
-    visual comparison.
-
-    **Signs of High Risk:**
-    * Discrepancies between the actual cluster size distribution and the predicted cluster size distribution may
-    indicate high risk.
-    * An irregular distribution of data across clusters in the predicted outcomes points towards an inaccurate
-    prediction model.
-    * A high number of outlier clusters could indicate that the model has trouble correctly grouping data.
-
-    **Strengths:**
-    * `ClusterSizeDistribution` provides a visual and intuitive way to compare the performance of the clustering model
-    against the actual data.
-    * This metric can effectively reveal where the model might be over- or underestimating cluster sizes.
-    * It works well with any clustering models, making it a versatile comparison tool.
-
-    **Limitations:**
-    * The metric assumes that the actual cluster distribution is optimal, which may not always be the case.
-    * It relies heavily on visual comparison, which might be subjective and may not provide a precise numerical measure
-    of model performance.
-    * The metric might not fully capture other important aspects of clustering such as cluster density, distances
-    between clusters, and the shape of clusters.
+    Assesses the performance of clustering models by comparing the distribution of cluster sizes in model predictions
+    with the actual data.
+
+    ### Purpose
+
+    The Cluster Size Distribution test aims to assess the performance of clustering models by comparing the
+    distribution of cluster sizes in the model's predictions with the actual data. This comparison helps determine if
+    the clustering model's output aligns well with the true cluster distribution, providing insights into the model's
+    accuracy and performance.
+
+    ### Test Mechanism
+
+    The test mechanism involves the following steps:
+    - Run the clustering model on the provided dataset to obtain predictions.
+    - Convert both the actual and predicted outputs into pandas dataframes.
+    - Use pandas built-in functions to derive the cluster size distributions from these dataframes.
+    - Construct two histograms: one for the actual cluster size distribution and one for the predicted distribution.
+    - Plot the histograms side-by-side for visual comparison.
+
+    ### Signs of High Risk
+
+    - Discrepancies between the actual cluster size distribution and the predicted cluster size distribution.
+    - Irregular distribution of data across clusters in the predicted outcomes.
+    - High number of outlier clusters suggesting the model struggles to correctly group data.
+
+    ### Strengths
+
+    - Provides a visual and intuitive way to compare the clustering model's performance against actual data.
+    - Effectively reveals where the model may be over- or underestimating cluster sizes.
+    - Versatile as it works well with any clustering model.
+
+    ### Limitations
+
+    - Assumes that the actual cluster distribution is optimal, which may not always be the case.
+    - Relies heavily on visual comparison, which could be subjective and may not offer a precise numerical measure of
+    performance.
+    - May not fully capture other important aspects of clustering, such as cluster density, distances between clusters,
+    and the shape of clusters.
     """
 
     name = "cluster_size_distribution"

validmind/tests/model_validation/ContextualRecall.py

@@ -13,25 +13,47 @@ from validmind import tags, tasks
 @tasks("text_classification", "text_summarization")
 def ContextualRecall(dataset, model):
     """
-    Evaluates a Natural Language Generation model's ability to generate contextually relevant and factually correct text, visualizing the results through histograms and bar charts, alongside compiling a comprehensive table of descriptive statistics for contextual recall scores.
+    Evaluates a Natural Language Generation model's ability to generate contextually relevant and factually correct
+    text, visualizing the results through histograms and bar charts, alongside compiling a comprehensive table of
+    descriptive statistics for contextual recall scores.
 
-    **Purpose:**
-    The Contextual Recall metric is used to evaluate the ability of a natural language generation (NLG) model to generate text that appropriately reflects the given context or prompt. It measures the model's capability to remember and reproduce the main context in its resulting output. This metric is critical in natural language processing tasks, as the coherency and contextuality of the generated text are essential.
+    ### Purpose
 
-    **Test Mechanism:**
-    The function starts by extracting the true and predicted values from the provided dataset and model. It then tokenizes the reference and candidate texts into discernible words or tokens using NLTK. The token overlap between the reference and candidate texts is identified, and the Contextual Recall score is computed by dividing the number of overlapping tokens by the total number of tokens in the reference text. Scores are calculated for each test dataset instance, resulting in an array of scores. These scores are visualized using a histogram and a bar chart to show score variations across different rows. Additionally, a table of descriptive statistics (mean, median, standard deviation, minimum, and maximum) is compiled for the contextual recall scores, providing a comprehensive summary of the model's performance.
+    The Contextual Recall metric is used to evaluate the ability of a natural language generation (NLG) model to
+    generate text that appropriately reflects the given context or prompt. It measures the model's capability to
+    remember and reproduce the main context in its resulting output. This metric is critical in natural language
+    processing tasks, as the coherency and contextuality of the generated text are essential.
 
-    **Signs of High Risk:**
-    - Low contextual recall scores could indicate that the model is not effectively reflecting the original context in its output, leading to incoherent or contextually misaligned text.
+    ### Test Mechanism
+
+    The function starts by extracting the true and predicted values from the provided dataset and model. It then
+    tokenizes the reference and candidate texts into discernible words or tokens using NLTK. The token overlap between
+    the reference and candidate texts is identified, and the Contextual Recall score is computed by dividing the number
+    of overlapping tokens by the total number of tokens in the reference text. Scores are calculated for each test
+    dataset instance, resulting in an array of scores. These scores are visualized using a histogram and a bar chart to
+    show score variations across different rows. Additionally, a table of descriptive statistics (mean, median,
+    standard deviation, minimum, and maximum) is compiled for the contextual recall scores, providing a comprehensive
+    summary of the model's performance.
+
+    ### Signs of High Risk
+
+    - Low contextual recall scores could indicate that the model is not effectively reflecting the original context in
+    its output, leading to incoherent or contextually misaligned text.
     - A consistent trend of low recall scores could suggest underperformance of the model.
 
-    **Strengths:**
-    - Provides a quantifiable measure of a model's adherence to the context and factual elements of the generated narrative.
-    - Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of contextual recall scores.
-    - Descriptive statistics offer a concise summary of the model's performance in generating contextually relevant texts.
+    ### Strengths
+
+    - Provides a quantifiable measure of a model's adherence to the context and factual elements of the generated
+    narrative.
+    - Visual representations (histograms and bar charts) make it easier to interpret the distribution and trends of
+    contextual recall scores.
+    - Descriptive statistics offer a concise summary of the model's performance in generating contextually relevant
+    texts.
+
+    ### Limitations
 
-    **Limitations:**
-    - The focus on word overlap could result in high scores for texts that use many common words, even when these texts lack coherence or meaningful context.
+    - The focus on word overlap could result in high scores for texts that use many common words, even when these texts
+    lack coherence or meaningful context.
     - This metric does not consider the order of words, which could lead to overestimated scores for scrambled outputs.
     - Models that effectively use infrequent words might be undervalued, as these words might not overlap as often.
     """

validmind/tests/model_validation/FeaturesAUC.py

@@ -19,24 +19,43 @@ logger = get_logger(__name__)
 @dataclass
 class FeaturesAUC(Metric):
     """
-    Evaluates the discriminatory power of each individual feature within a binary classification model by calculating the Area Under the Curve (AUC) for each feature separately.
+    Evaluates the discriminatory power of each individual feature within a binary classification model by calculating
+    the Area Under the Curve (AUC) for each feature separately.
 
-    **Purpose**: The central objective of this metric is to quantify how well each feature on its own can differentiate between the two classes in a binary classification problem. It serves as a univariate analysis tool that can help in pre-modeling feature selection or post-modeling interpretation.
+    ### Purpose
 
-    **Test Mechanism**: For each feature, the metric treats the feature values as raw scores to compute the AUC against the actual binary outcomes. It provides an AUC value for each feature, offering a simple yet powerful indication of each feature's univariate classification strength.
+    The central objective of this metric is to quantify how well each feature on its own can differentiate between the
+    two classes in a binary classification problem. It serves as a univariate analysis tool that can help in
+    pre-modeling feature selection or post-modeling interpretation.
 
-    **Signs of High Risk**:
-    - A feature with a low AUC score may not be contributing significantly to the differentiation between the two classes, which could be a concern if it is expected to be predictive.
-    - Conversely, a surprisingly high AUC for a feature not believed to be informative may suggest data leakage or other issues with the data.
+    ### Test Mechanism
 
-    **Strengths**:
-    - By isolating each feature, it highlights the individual contribution of features to the classification task without the influence of other variables.
-    - Useful for both initial feature evaluation and for providing insights into the model's reliance on individual features after model training.
+    For each feature, the metric treats the feature values as raw scores to compute the AUC against the actual binary
+    outcomes. It provides an AUC value for each feature, offering a simple yet powerful indication of each feature's
+    univariate classification strength.
 
-    **Limitations**:
-    - Does not reflect the combined effects of features or any interaction between them, which can be critical in certain models.
-    - The AUC values are calculated without considering the model's use of the features, which could lead to different interpretations of feature importance when considering the model holistically.
-    - This metric is applicable only to binary classification tasks and cannot be directly extended to multiclass classification or regression without modifications.
+    ### Signs of High Risk
+
+    - A feature with a low AUC score may not be contributing significantly to the differentiation between the two
+    classes, which could be a concern if it is expected to be predictive.
+    - Conversely, a surprisingly high AUC for a feature not believed to be informative may suggest data leakage or
+    other issues with the data.
+
+    ### Strengths
+
+    - By isolating each feature, it highlights the individual contribution of features to the classification task
+    without the influence of other variables.
+    - Useful for both initial feature evaluation and for providing insights into the model's reliance on individual
+    features after model training.
+
+    ### Limitations
+
+    - Does not reflect the combined effects of features or any interaction between them, which can be critical in
+    certain models.
+    - The AUC values are calculated without considering the model's use of the features, which could lead to different
+    interpretations of feature importance when considering the model holistically.
+    - This metric is applicable only to binary classification tasks and cannot be directly extended to multiclass
+    classification or regression without modifications.
     """
 
     name = "features_auc"