validmind 2.0.1__py3-none-any.whl → 2.0.7__py3-none-any.whl

validmind/tests/model_validation/RougeMetrics.py

@@ -76,7 +76,6 @@ class RougeMetrics(Metric):
         if r_metrics is None:
             raise ValueError("rouge_metrics must be provided in params")
 
-        # With all
         if not (
             set(self.default_params.get("rouge_metrics")).intersection(r_metrics)
             == set(r_metrics)
@@ -97,12 +96,13 @@ class RougeMetrics(Metric):
 
         metrics_df = pd.DataFrame(score_list)
         figures = []
+
         for m in metrics_df.columns:
             df_scores = pd.DataFrame(metrics_df[m].tolist())
             # Visualization part
             fig = go.Figure()
 
-            # Adding the line plots
+            # Adding the line plots for precision, recall, and F1-score with lines and markers
             fig.add_trace(
                 go.Scatter(
                     x=df_scores.index,
@@ -129,11 +129,13 @@ class RougeMetrics(Metric):
             )
 
             fig.update_layout(
-                title="ROUGE Scores for Each Row",
+                title=f"ROUGE Scores for {m}",
                 xaxis_title="Row Index",
                 yaxis_title="Score",
             )
-            k = m.replace("-", "")
+
+            # Ensure a unique key for each metric
+            k = f"{m.replace('-', '')}_{len(figures)}"
             figures.append(
                 Figure(
                     for_object=self,

validmind/tests/model_validation/SelfCheckNLIScore.py

@@ -0,0 +1,112 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import itertools
+from dataclasses import dataclass
+
+import pandas as pd
+import plotly.graph_objects as go
+import torch
+from selfcheckgpt.modeling_selfcheck import SelfCheckNLI
+from tqdm import tqdm
+
+from validmind.vm_models import Figure, Metric
+
+
+@dataclass
+class SelfCheckNLIScore(Metric):
+    """
+    Evaluates text generation models' performance by quantifying the level of hallucination in generated texts compared to reference texts.
+
+    **Purpose**: The HallucinationScore metric is designed to assess the factual accuracy and reliability of text generated by models, focusing on the detection and quantification of hallucinations—instances where generated content deviates from factual or expected outputs. By comparing generated texts against reference texts, this metric highlights discrepancies indicative of hallucinations, offering insights into the model's ability to produce contextually and factually coherent content.
+
+    **Test Mechanism**: To compute the HallucinationScore, the metric employs a comparison between the generated texts (model predictions) and the provided reference texts (true values). Using the SelfCheckNLI model, it evaluates each generated text's level of factual congruence with the reference, assigning a hallucination score based on the semantic coherence and factual accuracy. The scores for each text instance are then visualized in a line plot, allowing for the examination of hallucination trends across the dataset.
+
+    **Signs of High Risk**:
+    - High hallucination scores across a significant portion of the dataset, indicating a prevalence of factually inaccurate or irrelevant content generation.
+    - Patterns of consistent hallucination in specific contexts or subjects, suggesting gaps in the model's understanding or knowledge.
+    - Sharp fluctuations in hallucination scores, which may reveal inconsistencies in the model's performance or sensitivity to certain types of input.
+
+    **Strengths**:
+    - Directly addresses the critical aspect of factual accuracy in generated text, beyond mere linguistic or stylistic coherence.
+    - Provides a granular, instance-by-instance analysis of model performance, allowing for targeted improvements and diagnostics.
+    - Facilitates a deeper understanding of a model's capabilities and limitations in producing reliable and accurate content.
+
+    **Limitations**:
+    - Reliance on the SelfCheckNLI model means the accuracy and effectiveness of the HallucinationScore are contingent upon the performance and suitability of the underlying NLI model.
+    - May not fully capture the subtleties of certain factual inaccuracies or the contextual relevance of reference texts, especially in complex or nuanced domains.
+    - Potentially resource-intensive, given the computational demands of running advanced NLI models for large datasets.
+    """
+
+    name = "self_check_nli_score"
+    required_inputs = ["model", "dataset"]
+
+    def run(self):
+        # Assuming the dataset is structured with generated sentences and reference samples
+        y_true = list(itertools.chain.from_iterable(self.inputs.dataset.y))
+        y_pred = self.inputs.dataset.y_pred(self.inputs.model.input_id)
+
+        hallucination_scores = self.compute_hallucination_scores(y_pred, y_true)
+
+        # Visualization of scores
+        figures = self.visualize_scores(hallucination_scores)
+
+        return self.cache_results(figures=figures)
+
+    def compute_hallucination_scores(self, predictions, references):
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        selfcheck_nli = SelfCheckNLI(device=device)
+        hallucination_scores = []
+
+        print("Starting hallucination score computation...")
+
+        for index, (sentences, samples) in enumerate(
+            tqdm(zip(predictions, references), total=len(predictions))
+        ):
+            sent_scores_nli = selfcheck_nli.predict(
+                sentences=sentences, sampled_passages=samples
+            )
+
+            # Compute the mean of the hallucination scores for this row
+            average_score = sent_scores_nli.mean()
+            hallucination_scores.append(average_score)
+
+            # Print a progress update for each row
+            print(
+                f"Row {index + 1}/{len(predictions)}: Average hallucination score: {average_score}"
+            )
+
+        print("Completed hallucination score computation.")
+
+        return hallucination_scores
+
+    def visualize_scores(self, scores):
+        scores_df = pd.DataFrame(scores, columns=["Hallucination Score"])
+
+        fig = go.Figure()
+        fig.add_trace(
+            go.Scatter(
+                x=scores_df.index,
+                y=scores_df["Hallucination Score"],
+                mode="lines+markers",
+                name="Hallucination Score",
+            )
+        )
+
+        fig.update_layout(
+            title="Hallucination Scores Across Text Instances",
+            xaxis_title="Text Instance Index",
+            yaxis_title="Hallucination Score",
+        )
+
+        # Wrapping the plotly figure for compatibility with your framework might be needed
+        figures = [
+            Figure(
+                for_object=self,
+                key=self.key,
+                figure=fig,
+            )
+        ]
+
+        return figures

validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py

@@ -59,30 +59,25 @@ class DescriptiveAnalytics(Metric):
     }
 
     def run(self):
-        mean = np.mean(self.inputs.dataset.y_pred(self.inputs.model.input_id))
-        median = np.median(self.inputs.dataset.y_pred(self.inputs.model.input_id))
-        std = np.std(self.inputs.dataset.y_pred(self.inputs.model.input_id))
+        # Assuming y_pred returns a 2D array of embeddings [samples, features]
+        preds = self.inputs.dataset.y_pred(self.inputs.model.input_id)
+
+        # Calculate statistics across the embedding dimensions, not across all embeddings
+        means = np.mean(preds, axis=0)  # Mean of each feature across all samples
+        medians = np.median(preds, axis=0)  # Median of each feature across all samples
+        stds = np.std(preds, axis=0)  # Std. dev. of each feature across all samples
+
+        # Plot histograms of the calculated statistics
+        mean_fig = px.histogram(x=means, title="Distribution of Embedding Means")
+        median_fig = px.histogram(x=medians, title="Distribution of Embedding Medians")
+        std_fig = px.histogram(
+            x=stds, title="Distribution of Embedding Standard Deviations"
+        )
 
         return self.cache_results(
             figures=[
-                Figure(
-                    for_object=self,
-                    key=self.key,
-                    figure=px.histogram(mean, title="Distribution of Embedding Means"),
-                ),
-                Figure(
-                    for_object=self,
-                    key=self.key,
-                    figure=px.histogram(
-                        median, title="Distribution of Embedding Medians"
-                    ),
-                ),
-                Figure(
-                    for_object=self,
-                    key=self.key,
-                    figure=px.histogram(
-                        std, title="Distribution of Embedding Standard Deviations"
-                    ),
-                ),
+                Figure(for_object=self, key=f"{self.key}_mean", figure=mean_fig),
+                Figure(for_object=self, key=f"{self.key}_median", figure=median_fig),
+                Figure(for_object=self, key=f"{self.key}_std", figure=std_fig),
             ],
         )

validmind/tests/model_validation/sklearn/ClassifierPerformance.py

@@ -131,7 +131,9 @@ class ClassifierPerformance(Metric):
         y_true = self.y_true()
         class_pred = self.y_pred()
 
-        report = metrics.classification_report(y_true, class_pred, output_dict=True)
+        report = metrics.classification_report(
+            y_true, class_pred, output_dict=True, zero_division=0
+        )
         report["roc_auc"] = multiclass_roc_auc_score(y_true, class_pred)
 
         return self.cache_results(report)

validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py

@@ -8,6 +8,7 @@ from dataclasses import dataclass
 import matplotlib.pyplot as plt
 import shap
 
+from validmind.errors import UnsupportedModelForSHAPError
 from validmind.logging import get_logger
 from validmind.vm_models import Figure, Metric
 
@@ -72,6 +73,9 @@ class SHAPGlobalImportance(Metric):
             "visualization",
         ],
     }
+    default_params = {
+        "kernel_explainer_samples": 10,
+    }
 
     def _generate_shap_plot(self, type_, shap_values, x_test):
         """
@@ -127,22 +131,44 @@ class SHAPGlobalImportance(Metric):
             model_class == "XGBClassifier"
             or model_class == "RandomForestClassifier"
             or model_class == "CatBoostClassifier"
+            or model_class == "DecisionTreeClassifier"
         ):
             explainer = shap.TreeExplainer(trained_model)
         elif (
             model_class == "LogisticRegression"
             or model_class == "XGBRegressor"
             or model_class == "LinearRegression"
+            or model_class == "LinearSVC"
         ):
             explainer = shap.LinearExplainer(trained_model, self.inputs.dataset.x)
+        elif model_class == "SVC":
+            # KernelExplainer is slow so we use shap.sample to speed it up
+            explainer = shap.KernelExplainer(
+                trained_model.predict,
+                shap.sample(
+                    self.inputs.dataset.x,
+                    self.params["kernel_explainer_samples"],
+                ),
+            )
+        else:
+            raise UnsupportedModelForSHAPError(
+                f"Model {model_class} not supported for SHAP importance."
+            )
+
+        # KernelExplainer is slow so we use shap.sample to speed it up
+        if isinstance(explainer, shap.KernelExplainer):
+            shap_sample = shap.sample(
+                self.inputs.dataset.x,
+                self.params["kernel_explainer_samples"],
+            )
         else:
-            raise ValueError(f"Model {model_class} not supported for SHAP importance.")
+            shap_sample = self.inputs.dataset.x
 
-        shap_values = explainer.shap_values(self.inputs.dataset.x)
+        shap_values = explainer.shap_values(shap_sample)
 
         figures = [
-            self._generate_shap_plot("mean", shap_values, self.inputs.dataset.x),
-            self._generate_shap_plot("summary", shap_values, self.inputs.dataset.x),
+            self._generate_shap_plot("mean", shap_values, shap_sample),
+            self._generate_shap_plot("summary", shap_values, shap_sample),
         ]
 
         # restore warnings

validmind/tests/model_validation/sklearn/TrainingTestDegradation.py

@@ -129,12 +129,12 @@ class TrainingTestDegradation(ThresholdTest):
         y_test_true = y_test_true.astype(y_test_pred.dtype)
 
         report_train = metrics.classification_report(
-            y_train_true, y_train_pred, output_dict=True
+            y_train_true, y_train_pred, output_dict=True, zero_division=0
         )
         report_train["roc_auc"] = multiclass_roc_auc_score(y_train_true, y_train_pred)
 
         report_test = metrics.classification_report(
-            y_test_true, y_test_pred, output_dict=True
+            y_test_true, y_test_pred, output_dict=True, zero_division=0
         )
         report_test["roc_auc"] = multiclass_roc_auc_score(y_test_true, y_test_pred)
 
@@ -145,7 +145,13 @@ class TrainingTestDegradation(ThresholdTest):
             for metric_name in ["precision", "recall", "f1-score"]:
                 train_score = report_train[class_name][metric_name]
                 test_score = report_test[class_name][metric_name]
-                degradation = (train_score - test_score) / train_score
+
+                # If training score is 0, degradation is assumed to be 100%
+                if train_score == 0:
+                    degradation = 1.0
+                else:
+                    degradation = (train_score - test_score) / train_score
+
                 passed = degradation < self.params["max_threshold"]
                 test_results.append(
                     ThresholdTestResult(

validmind/tests/model_validation/statsmodels/RegressionModelsPerformance.py

@@ -79,7 +79,7 @@ class RegressionModelsPerformance(Metric):
     def sample_performance_ols(self, models, datasets):
         evaluation_results = []
 
-        for (model, dataset) in zip(models, datasets):
+        for model, dataset in zip(models, datasets):
             X_columns = dataset.get_features_columns()
             y_true = dataset.y
             y_pred = dataset.y_pred(model.input_id)

validmind/tests/prompt_validation/ai_powered_test.py

@@ -57,6 +57,8 @@ class AIPoweredTest:
                     {"role": "system", "content": system_prompt},
                     {"role": "user", "content": user_prompt},
                 ],
+                temperature=0.0,
+                seed=42,
             )
             .choices[0]
             .message.content

validmind/unit_metrics/__init__.py

@@ -237,8 +237,6 @@ def run_metric(metric_id=None, inputs=None, params=None):
     # Run the metric
     result = metric.run()
 
-    cache_key = get_metric_cache_key(metric_id, params, inputs)
-
     unit_metric_results_cache[cache_key] = result
 
     return result

validmind/unit_metrics/composite.py

@@ -0,0 +1,275 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import ast
+import inspect
+from dataclasses import dataclass
+from typing import List
+from uuid import uuid4
+
+from ..errors import LoadTestError
+from ..logging import get_logger
+from ..utils import clean_docstring, run_async, test_id_to_name
+from ..vm_models.test.metric import Metric
+from ..vm_models.test.metric_result import MetricResult
+from ..vm_models.test.result_summary import ResultSummary, ResultTable
+from ..vm_models.test.result_wrapper import MetricResultWrapper
+from . import _get_metric_class, run_metric
+
+logger = get_logger(__name__)
+
+
+def _extract_class_methods(cls):
+    source = inspect.getsource(cls)
+    tree = ast.parse(source)
+
+    class MethodVisitor(ast.NodeVisitor):
+        def __init__(self):
+            self.methods = {}
+
+        def visit_FunctionDef(self, node):
+            self.methods[node.name] = node
+            self.generic_visit(node)
+
+    visitor = MethodVisitor()
+    visitor.visit(tree)
+
+    return visitor.methods
+
+
+def _extract_required_inputs(cls):
+    methods = _extract_class_methods(cls)
+
+    class Visitor(ast.NodeVisitor):
+        def __init__(self):
+            self.properties = set()
+            self.visited_methods = set()
+
+        def visit_Attribute(self, node):
+            if isinstance(node.value, ast.Attribute) and node.value.attr == "inputs":
+                self.properties.add(node.attr)
+
+            self.generic_visit(node)
+
+        def visit_Call(self, node):
+            if isinstance(node.func, ast.Attribute) and isinstance(
+                node.func.value, ast.Name
+            ):
+                if node.func.value.id == "self" and node.func.attr in methods:
+                    method_name = node.func.attr
+
+                    if method_name not in self.visited_methods:
+                        self.visited_methods.add(method_name)
+                        self.visit(methods[method_name])
+
+            self.generic_visit(node)
+
+    visitor = Visitor()
+    visitor.visit(methods["run"])
+
+    return visitor.properties
+
+
+@dataclass
+class CompositeMetric(Metric):
+    unit_metrics: List[str] = None
+
+    def __post_init__(self):
+        if self._unit_metrics:
+            self.unit_metrics = self._unit_metrics
+        elif self.unit_metrics is None:
+            raise ValueError("unit_metrics must be provided")
+
+        if hasattr(self, "_output_template") and self._output_template:
+            self.output_template = self._output_template
+
+    def run(self):
+        self.result = run_metrics(
+            test_id=self.test_id,
+            metric_ids=self.unit_metrics,
+            description=self.description(),
+            inputs=self._get_input_dict(),
+            params=self.params,
+            output_template=self.output_template,
+            show=False,
+        )
+
+        return self.result
+
+    def summary(self, result: dict):
+        return ResultSummary(results=[ResultTable(data=[result])])
+
+
+def load_composite_metric(
+    test_id: str = None,
+    metric_name: str = None,
+    unit_metrics: List[str] = None,
+    output_template: str = None,
+) -> CompositeMetric:
+    # this function can either create a composite metric from a list of unit metrics or
+    # load a stored composite metric based on the test id
+
+    # TODO: figure out this circular import thing:
+    from ..api_client import get_metadata
+
+    if test_id:
+        # get the unit metric ids and output template (if any) from the metadata
+        try:
+            unit_metrics = run_async(
+                get_metadata, f"composite_metric_def:{test_id}:unit_metrics"
+            )["json"]
+            output_template = run_async(
+                get_metadata, f"composite_metric_def:{test_id}:output_template"
+            )["json"]["output_template"]
+        except Exception:
+            logger.error(f"Could not load composite metric {test_id}")
+            raise LoadTestError(f"Could not load composite metric {test_id}")
+
+    description = f"""
+    Composite metric built from the following unit metrics:
+    {', '.join([metric_id.split('.')[-1] for metric_id in unit_metrics])}
+    """
+
+    class_def = type(
+        test_id.split(".")[-1] if test_id else metric_name,
+        (CompositeMetric,),
+        {
+            "__doc__": description,
+            "_unit_metrics": unit_metrics,
+            "_output_template": output_template,
+        },
+    )
+
+    required_inputs = set()
+    for metric_id in unit_metrics:
+        metric_cls = _get_metric_class(metric_id)
+        # required_inputs.update(_extract_required_inputs(metric_cls))
+        required_inputs.update(metric_cls.required_inputs or [])
+
+    class_def.required_inputs = list(required_inputs)
+
+    return class_def
+
+
+def run_metrics(
+    name: str = None,
+    metric_ids: List[str] = None,
+    description: str = None,
+    output_template: str = None,
+    inputs: dict = None,
+    params: dict = None,
+    test_id: str = None,
+    show: bool = True,
+) -> MetricResultWrapper:
+    """Run a composite metric
+
+    Composite metrics are metrics that are composed of multiple unit metrics. This
+    works by running individual unit metrics and then combining the results into a
+    single "MetricResult" object that can be logged and displayed just like any other
+    metric result. The special thing about composite metrics is that when they are
+    logged to the platform, metadata describing the unit metrics and output template
+    used to generate the composite metric is also logged. This means that by grabbing
+    the metadata for a composite metric (identified by the test ID
+    `validmind.composite_metric.<name>`) the framework can rebuild and rerun it at
+    any time.
+
+    Args:
+        name (str, optional): Name of the composite metric. Required if test_id is not
+            provided. Defaults to None.
+        metric_ids (list[str]): List of unit metric IDs to run. Required.
+        description (str, optional): Description of the composite metric. Defaults to
+            None.
+        output_template (_type_, optional): Output template to customize the result
+            table.
+        inputs (_type_, optional): Inputs to pass to the unit metrics. Defaults to None
+        params (_type_, optional): Parameters to pass to the unit metrics. Defaults to
+            None.
+        test_id (str, optional): Test ID of the composite metric. Required if name is
+            not provided. Defaults to None.
+        show (bool, optional): Whether to show the result immediately. Defaults to True
+
+    Raises:
+        ValueError: If metric_ids is not provided
+        ValueError: If name or key is not provided
+
+    Returns:
+        MetricResultWrapper: The result wrapper object
+    """
+    if not metric_ids:
+        raise ValueError("metric_ids must be provided")
+
+    if not name and not test_id:
+        raise ValueError("name or key must be provided")
+
+    # if name is provided, make sure to squash it into a camel case string
+    if name:
+        name = "".join(word[0].upper() + word[1:] for word in name.split())
+
+    results = {}
+
+    for metric_id in metric_ids:
+        result = run_metric(
+            metric_id=metric_id,
+            inputs=inputs,
+            params=params,
+        )
+        results[list(result.summary.keys())[0]] = result.value
+
+    test_id = f"validmind.composite_metric.{name}" if not test_id else test_id
+
+    if not output_template:
+
+        def row(key):
+            return f"""
+            <tr>
+                <td><strong>{key.upper()}</strong></td>
+                <td>{{{{ value['{key}'] | number }}}}</td>
+            </tr>
+            """
+
+        output_template = f"""
+        <h1{test_id_to_name(test_id)}</h1>
+        <table>
+            <thead>
+                <tr>
+                    <th>Metric</th>
+                    <th>Value</th>
+                </tr>
+            </thead>
+            <tbody>
+                {"".join([row(key) for key in results.keys()])}
+            </tbody>
+        </table>
+        """
+
+    result_wrapper = MetricResultWrapper(
+        result_id=test_id,
+        result_metadata=[
+            {
+                "content_id": f"metric_description:{test_id}",
+                "text": clean_docstring(description),
+            },
+            {
+                "content_id": f"composite_metric_def:{test_id}:unit_metrics",
+                "json": metric_ids,
+            },
+            {
+                "content_id": f"composite_metric_def:{test_id}:output_template",
+                "json": {"output_template": output_template},
+            },
+        ],
+        inputs=list(inputs.keys()),
+        output_template=output_template,
+        metric=MetricResult(
+            key=test_id,
+            ref_id=str(uuid4()),
+            value=results,
+            summary=ResultSummary(results=[ResultTable(data=[results])]),
+        ),
+    )
+
+    if show:
+        result_wrapper.show()
+
+    return result_wrapper

validmind/unit_metrics/regression/GiniCoefficient.py

@@ -0,0 +1,39 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from dataclasses import dataclass
+
+import numpy as np
+
+from validmind.vm_models import UnitMetric
+
+
+@dataclass
+class GiniCoefficient(UnitMetric):
+    required_inputs = ["dataset", "model"]
+
+    def run(self):
+        y_true = self.inputs.dataset.y
+        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
+
+        # Sort true values and corresponding predicted values
+        idx = np.argsort(y_true)
+        y_true_sorted = y_true[idx]
+        y_pred_sorted = y_pred[idx]
+
+        # Compute cumulative sums
+        cumsum_true = np.cumsum(y_true_sorted)
+        cumsum_pred = np.cumsum(y_pred_sorted)
+
+        # Normalize cumulative sums
+        cumsum_true_norm = cumsum_true / np.max(cumsum_true)
+        cumsum_pred_norm = cumsum_pred / np.max(cumsum_pred)
+
+        # Compute area under the Lorenz curve
+        area_lorenz = np.trapz(cumsum_pred_norm, x=cumsum_true_norm)
+
+        # Compute Gini coefficient
+        gini_coeff = 1 - 2 * area_lorenz
+
+        return self.cache_results(metric_value=gini_coeff)

validmind/unit_metrics/regression/HuberLoss.py

@@ -0,0 +1,27 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from dataclasses import dataclass
+
+import numpy as np
+
+from validmind.vm_models import UnitMetric
+
+
+@dataclass
+class HuberLoss(UnitMetric):
+    required_inputs = ["dataset", "model"]
+
+    def run(self):
+        y_true = self.inputs.dataset.y
+        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
+
+        # delta - Threshold for the squared error to be linear or quadratic.
+        delta = 1.0
+        error = y_true - y_pred
+        quadratic_part = np.minimum(np.abs(error), delta)
+        linear_part = np.abs(error) - quadratic_part
+        value = np.mean(0.5 * quadratic_part**2 + delta * linear_part)
+
+        return self.cache_results(metric_value=value)