validmind 2.2.6__py3-none-any.whl → 2.3.1__py3-none-any.whl

validmind/tests/model_validation/ragas/Faithfulness.py

@@ -11,7 +11,7 @@ from ragas.metrics import faithfulness
 
 from validmind import tags, tasks
 
-from .utils import get_renamed_columns
+from .utils import get_ragas_config, get_renamed_columns
 
 
 @tags("ragas", "llm", "rag_performance")
@@ -20,7 +20,7 @@ def Faithfulness(
     dataset,
     answer_column="answer",
     contexts_column="contexts",
-):
+):  # noqa
     """
     Evaluates the faithfulness of the generated answers with respect to retrieved contexts.
 
@@ -93,8 +93,7 @@ def Faithfulness(
     df = get_renamed_columns(dataset.df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df),
-        metrics=[faithfulness],
+        Dataset.from_pandas(df), metrics=[faithfulness], **get_ragas_config()
     ).to_pandas()
 
     fig_histogram = px.histogram(x=result_df["faithfulness"].to_list(), nbins=10)
@@ -102,7 +101,9 @@ def Faithfulness(
 
     return (
         {
-            "Scores": result_df[["contexts", "answer", "faithfulness"]],
+            "Scores (will not be uploaded to UI)": result_df[
+                ["contexts", "answer", "faithfulness"]
+            ],
             "Aggregate Scores": [
                 {
                     "Mean Score": result_df["faithfulness"].mean(),

validmind/tests/model_validation/ragas/utils.py

@@ -2,17 +2,42 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import os
 
-def _udf_get_sub_col(x, root_col, sub_col):
-    if not isinstance(x, dict):
-        raise TypeError(f"Expected a dictionary in column '{root_col}', got {type(x)}.")
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
 
-    if sub_col not in x:
-        raise KeyError(
-            f"Sub-column '{sub_col}' not found in dictionary in column '{root_col}'."
-        )
+from validmind.ai.utils import get_client_and_model
 
-    return x[sub_col]
+EMBEDDINGS_MODEL = "text-embedding-3-small"
+
+
+def get_ragas_config():
+    client, model = get_client_and_model()
+    os.environ["OPENAI_API_BASE"] = str(client.base_url)
+
+    return {
+        "llm": ChatOpenAI(api_key=client.api_key, model=model),
+        "embeddings": OpenAIEmbeddings(api_key=client.api_key, model=EMBEDDINGS_MODEL),
+    }
+
+
+def make_sub_col_udf(root_col, sub_col):
+    """Create a udf that extracts sub-column values from a dictionary."""
+
+    def _udf_get_sub_col(x):
+        if not isinstance(x, dict):
+            raise TypeError(
+                f"Expected a dictionary in column '{root_col}', got {type(x)}."
+            )
+
+        if sub_col not in x:
+            raise KeyError(
+                f"Sub-column '{sub_col}' not found in dictionary in column '{root_col}'."
+            )
+
+        return x[sub_col]
+
+    return _udf_get_sub_col
 
 
 def get_renamed_columns(df, column_map):
@@ -34,6 +59,7 @@ def get_renamed_columns(df, column_map):
     Returns:
         pd.DataFrame: The DataFrame with columns renamed.
     """
+
     new_df = df.copy()
 
     for new_name, source in column_map.items():
@@ -50,7 +76,7 @@ def get_renamed_columns(df, column_map):
 
             if root_col in new_df.columns:
                 new_df[new_name] = new_df[root_col].apply(
-                    lambda x: _udf_get_sub_col(x, root_col, sub_col)
+                    make_sub_col_udf(root_col, sub_col)
                 )
 
             else:

validmind/tests/model_validation/sklearn/ClusterPerformance.py

@@ -66,7 +66,7 @@ class ClusterPerformance(Metric):
         y_true_test = y_true_test.astype(y_pred_test.dtype).flatten()
         results = []
         for metric_name, metric_fcn in metric_info.items():
-            for sample in samples:
+            for _ in samples:
                 train_value = metric_fcn(list(y_true_train), y_pred_train)
                 test_value = metric_fcn(list(y_true_test), y_pred_test)
             results.append(
@@ -85,7 +85,7 @@ class ClusterPerformance(Metric):
         """
         table_records = []
         for result in raw_results:
-            for key, value in result.items():
+            for key, _ in result.items():
                 table_records.append(
                     {
                         "Metric": key,

validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.py

@@ -123,7 +123,7 @@ class ClusterPerformanceMetrics(ClusterPerformance):
         """
         table_records = []
         for result in raw_results:
-            for key, value in result.items():
+            for key, _ in result.items():
                 table_records.append(
                     {
                         "Metric": key,

validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.py

@@ -52,7 +52,7 @@ class ModelsPerformanceComparison(ClassifierPerformance):
     """
 
     name = "models_performance_comparison"
-    required_inputs = ["model", "models", "dataset"]
+    required_inputs = ["dataset", "models"]
     metadata = {
         "task_types": ["classification", "text_classification"],
         "tags": [
@@ -70,12 +70,12 @@ class ModelsPerformanceComparison(ClassifierPerformance):
         """
         results = []
         prf_table = []
-        classes = {str(i) for i in unique(self.y_true())}
+        classes = {str(i) for i in unique(self.inputs.dataset.y)}
 
         for class_name in classes:
             prf_dict = {}
             prf_dict["Class"] = class_name
-            for m, m_v in metric_value.items():
+            for m, _ in metric_value.items():
                 prf_dict[f"Precision- {m}"] = metric_value[m][class_name]["precision"]
                 prf_dict[f"Recall- {m}"] = metric_value[m][class_name]["recall"]
                 prf_dict[f"F1- {m}"] = metric_value[m][class_name]["f1-score"]
@@ -85,7 +85,7 @@ class ModelsPerformanceComparison(ClassifierPerformance):
         for class_name in avg_metrics:
             avg_dict = {}
             avg_dict["Class"] = class_name
-            for m, m_v in metric_value.items():
+            for m, _ in metric_value.items():
                 avg_dict[f"Precision- {m}"] = metric_value[m][class_name]["precision"]
                 avg_dict[f"Recall- {m}"] = metric_value[m][class_name]["recall"]
                 avg_dict[f"F1- {m}"] = metric_value[m][class_name]["f1-score"]
@@ -103,7 +103,7 @@ class ModelsPerformanceComparison(ClassifierPerformance):
         for metric_name in ["accuracy", "roc_auc"]:
             acc_roc_auc_dict = {}
             acc_roc_auc_dict["Metric"] = metric_name
-            for m, m_v in metric_value.items():
+            for m, _ in metric_value.items():
                 acc_roc_auc_dict[f"accuracy- {m}"] = metric_value[m]["accuracy"]
                 acc_roc_auc_dict[f"roc_auc- {m}"] = metric_value[m]["roc_auc"]
             acc_roc_auc_table.append(acc_roc_auc_dict)
@@ -122,10 +122,8 @@ class ModelsPerformanceComparison(ClassifierPerformance):
                 "List of models must be provided as a `models` parameter to compare performance"
             )
 
-        all_models = [self.inputs.model]
+        all_models = self.inputs.models
 
-        if self.inputs.models is not None:
-            all_models.extend(self.inputs.models)
         results = {}
         for idx, model in enumerate(all_models):
             y_true = self.inputs.dataset.y

validmind/tests/model_validation/sklearn/RegressionErrors.py

@@ -57,7 +57,7 @@ class RegressionErrors(Metric):
         """
         table_records = []
         for result in raw_results:
-            for key, value in result.items():
+            for key, _ in result.items():
                 table_records.append(
                     {
                         "Metric": key,

validmind/tests/model_validation/sklearn/RegressionModelsPerformanceComparison.py

@@ -9,8 +9,11 @@ import numpy as np
 from sklearn.metrics import mean_absolute_error, mean_squared_error
 
 from validmind.errors import SkipTestError
+from validmind.logging import get_logger
 from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
 
+logger = get_logger(__name__)
+
 
 @dataclass
 class RegressionModelsPerformanceComparison(Metric):
@@ -56,7 +59,7 @@ class RegressionModelsPerformanceComparison(Metric):
     """
 
     name = "models_performance_comparison"
-    required_inputs = ["model", "dataset"]
+    required_inputs = ["dataset", "models"]
 
     metadata = {
         "task_types": ["regression"],
@@ -76,8 +79,14 @@ class RegressionModelsPerformanceComparison(Metric):
         results["Mean Squared Error (MSE)"] = mse_test
         results["Root Mean Squared Error (RMSE)"] = np.sqrt(mse_test)
 
-        mape_test = np.mean(np.abs((y_true_test - y_pred_test) / y_true_test)) * 100
-        results["Mean Absolute Percentage Error (MAPE)"] = mape_test
+        if np.any(y_true_test == 0):
+            logger.warning(
+                "y_true_test contains zero values. Skipping MAPE calculation to avoid division by zero."
+            )
+            results["Mean Absolute Percentage Error (MAPE)"] = None
+        else:
+            mape_test = np.mean(np.abs((y_true_test - y_pred_test) / y_true_test)) * 100
+            results["Mean Absolute Percentage Error (MAPE)"] = mape_test
 
         mbd_test = np.mean(y_pred_test - y_true_test)
         results["Mean Bias Deviation (MBD)"] = mbd_test
@@ -94,7 +103,7 @@ class RegressionModelsPerformanceComparison(Metric):
         for metric_name in metrics:
             errors_dict = {}
             errors_dict["Errors"] = metric_name
-            for m, m_v in metric_value.items():
+            for m, _ in metric_value.items():
                 for metric in metrics:
                     res = re.findall(r"\(.*?\)", metric)
                     res[0][1:-1]
@@ -117,10 +126,7 @@ class RegressionModelsPerformanceComparison(Metric):
                 "List of models must be provided as a `models` parameter to compare performance"
             )
 
-        all_models = [self.inputs.model]
-
-        if self.inputs.models is not None:
-            all_models.extend(self.inputs.models)
+        all_models = self.inputs.models
 
         results = {}
 

validmind/tests/model_validation/sklearn/RegressionR2Square.py

@@ -57,7 +57,7 @@ class RegressionR2Square(Metric):
         """
         table_records = []
         for result in raw_results:
-            for key, value in result.items():
+            for key, _ in result.items():
                 table_records.append(
                     {
                         "Metric": key,

validmind/tests/model_validation/statsmodels/DurbinWatsonTest.py

@@ -53,7 +53,7 @@ class DurbinWatsonTest(Metric):
         """
         Calculates DB for each of the dataset features
         """
-        x_train = self.train_ds.df
+        x_train = self.inputs.dataset.df
         dw_values = {}
         for col in x_train.columns:
             dw_values[col] = durbin_watson(x_train[col].values)

validmind/tests/model_validation/statsmodels/GINITable.py

@@ -80,7 +80,7 @@ class GINITable(Metric):
         metrics_dict = {"Dataset": [], "AUC": [], "GINI": [], "KS": []}
 
         # Iterate over each dataset in the inputs
-        for i, dataset in enumerate(self.inputs.datasets):
+        for _, dataset in enumerate(self.inputs.datasets):
             dataset_label = (
                 dataset.input_id
             )  # Use input_id as the label for each dataset

validmind/tests/model_validation/statsmodels/JarqueBera.py

@@ -59,7 +59,7 @@ class JarqueBera(Metric):
         """
         Calculates JB for each of the dataset features
         """
-        x_train = self.inputs.dataset.df
+        x_train = self.inputs.dataset.df[self.inputs.dataset.feature_columns_numeric]
 
         jb_values = {}
         for col in x_train.columns:

validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py

@@ -87,7 +87,7 @@ class KolmogorovSmirnov(Metric):
         if data_distribution not in ["norm" or "exp"]:
             InvalidTestParametersError("Dist parameter must be either 'norm' or 'exp'")
 
-        x_train = self.inputs.dataset.df
+        x_train = self.inputs.dataset.df[self.inputs.dataset.feature_columns_numeric]
         ks_values = {}
         for col in x_train.columns:
             ks_stat, p_value = kstest_normal(x_train[col].values, data_distribution)

validmind/tests/model_validation/statsmodels/LJungBox.py

@@ -54,7 +54,7 @@ class LJungBox(Metric):
         """
         Calculates Ljung-Box test for each of the dataset features
         """
-        x_train = self.train_ds.df
+        x_train = self.inputs.dataset.df
 
         ljung_box_values = {}
         for col in x_train.columns:

validmind/tests/model_validation/statsmodels/Lilliefors.py

@@ -70,7 +70,7 @@ class Lilliefors(Metric):
         """
         Calculates Lilliefors test for each of the dataset features
         """
-        x_train = self.train_ds.df
+        x_train = self.inputs.dataset.df[self.inputs.dataset.feature_columns_numeric]
 
         lilliefors_values = {}
         for col in x_train.columns:

validmind/tests/model_validation/statsmodels/RegressionCoeffsPlot.py

@@ -8,6 +8,7 @@ import pandas as pd
 import plotly.graph_objects as go
 from scipy import stats
 
+from validmind.errors import SkipTestError
 from validmind.vm_models import Figure, Metric
 
 
@@ -115,6 +116,9 @@ class RegressionCoeffsPlot(Metric):
             all_models.extend(self.inputs.models)
 
         for i, model in enumerate(all_models):
+            if model.library != "statsmodels":
+                raise SkipTestError("Only statsmodels are supported for this metric")
+
             model_name = f"Model {i+1}"
 
             fig, metric_values = self.plot_coefficients_with_ci(model, model_name)

validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py

@@ -7,6 +7,7 @@ from dataclasses import dataclass
 import matplotlib.pyplot as plt
 import seaborn as sns
 
+from validmind.errors import SkipTestError
 from validmind.logging import get_logger
 from validmind.vm_models import Figure, Metric
 
@@ -82,10 +83,14 @@ class RegressionFeatureSignificance(Metric):
         # Initialize a list to store figures
         figures = []
 
-        for i, fitted_model in enumerate(model_list):
+        for i, model in enumerate(model_list):
+
+            if model.library != "statsmodels":
+                raise SkipTestError("Only statsmodels are supported for this metric")
+
             # Get the coefficients and p-values from the model
-            coefficients = fitted_model.model.params
-            pvalues = fitted_model.model.pvalues
+            coefficients = model.model.params
+            pvalues = model.model.pvalues
 
             # Sort the variables by p-value in ascending order
             sorted_idx = pvalues.argsort()
@@ -122,7 +127,7 @@ class RegressionFeatureSignificance(Metric):
                     for_object=self,
                     key=f"{self.key}:{i}",
                     figure=fig,
-                    metadata={"model": str(fitted_model.model)},
+                    metadata={"model": str(model.model)},
                 )
             )
             plt.close("all")

validmind/tests/model_validation/statsmodels/RegressionModelsCoeffs.py

@@ -73,9 +73,9 @@ class RegressionModelsCoeffs(Metric):
             raise ValueError("List of models must be provided in the models parameter")
 
         for model in self.inputs.models:
-            if model.class_ != "statsmodels" and model.class_ != "R":
+            if model.library != "statsmodels":
                 raise SkipTestError(
-                    "Only statsmodels and R models are supported for this metric"
+                    "Only statsmodels models are supported for this metric"
                 )
 
         coefficients = [m.regression_coefficients() for m in self.inputs.models]

validmind/tests/model_validation/statsmodels/RunsTest.py

@@ -59,7 +59,7 @@ class RunsTest(Metric):
         """
         Calculates the run test for each of the dataset features
         """
-        x_train = self.inputs.dataset.df
+        x_train = self.inputs.dataset.df[self.inputs.dataset.feature_columns_numeric]
 
         runs_test_values = {}
         for col in x_train.columns:

validmind/tests/model_validation/statsmodels/ShapiroWilk.py

@@ -53,7 +53,7 @@ class ShapiroWilk(Metric):
         """
         Calculates Shapiro-Wilk test for each of the dataset features.
         """
-        x_train = self.inputs.dataset.df
+        x_train = self.inputs.dataset.df[self.inputs.dataset.feature_columns_numeric]
         sw_values = {}
         for col in x_train.columns:
             sw_stat, sw_pvalue = stats.shapiro(x_train[col].values)

validmind/tests/prompt_validation/Bias.py

@@ -7,6 +7,7 @@ from typing import List
 
 import pandas as pd
 
+from validmind.errors import MissingRequiredTestInputError
 from validmind.vm_models import (
     ResultSummary,
     ResultTable,
@@ -15,11 +16,16 @@ from validmind.vm_models import (
     ThresholdTestResult,
 )
 
-from .ai_powered_test import AIPoweredTest
+from .ai_powered_test import (
+    call_model,
+    get_explanation,
+    get_score,
+    missing_prompt_message,
+)
 
 
 @dataclass
-class Bias(ThresholdTest, AIPoweredTest):
+class Bias(ThresholdTest):
     """
     Evaluates bias in a Large Language Model based on the order and distribution of exemplars in a prompt.
 
@@ -103,12 +109,6 @@ Prompt:
 """
 '''.strip()
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)  # Call ThresholdTest.__init__
-        AIPoweredTest.__init__(
-            self, *args, **kwargs
-        )  # Explicitly call AIPoweredTest.__init__
-
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         result = results[0]
         results_table = [
@@ -132,14 +132,17 @@ Prompt:
         )
 
     def run(self):
-        response = self.call_model(
+        if not hasattr(self.inputs.model, "prompt"):
+            raise MissingRequiredTestInputError(missing_prompt_message)
+
+        response = call_model(
             system_prompt=self.system_prompt,
             user_prompt=self.user_prompt.format(
                 prompt_to_test=self.inputs.model.prompt.template
             ),
         )
-        score = self.get_score(response)
-        explanation = self.get_explanation(response)
+        score = get_score(response)
+        explanation = get_explanation(response)
 
         passed = score > self.params["min_threshold"]
         results = [

validmind/tests/prompt_validation/Clarity.py

@@ -7,6 +7,7 @@ from typing import List
 
 import pandas as pd
 
+from validmind.errors import MissingRequiredTestInputError
 from validmind.vm_models import (
     ResultSummary,
     ResultTable,
@@ -15,11 +16,16 @@ from validmind.vm_models import (
     ThresholdTestResult,
 )
 
-from .ai_powered_test import AIPoweredTest
+from .ai_powered_test import (
+    call_model,
+    get_explanation,
+    get_score,
+    missing_prompt_message,
+)
 
 
 @dataclass
-class Clarity(ThresholdTest, AIPoweredTest):
+class Clarity(ThresholdTest):
     """
     Evaluates and scores the clarity of prompts in a Large Language Model based on specified guidelines.
 
@@ -93,12 +99,6 @@ Prompt:
 """
 '''.strip()
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)  # Call ThresholdTest.__init__
-        AIPoweredTest.__init__(
-            self, *args, **kwargs
-        )  # Explicitly call AIPoweredTest.__init__
-
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         result = results[0]
         results_table = [
@@ -122,14 +122,17 @@ Prompt:
         )
 
     def run(self):
-        response = self.call_model(
+        if not hasattr(self.inputs.model, "prompt"):
+            raise MissingRequiredTestInputError(missing_prompt_message)
+
+        response = call_model(
             system_prompt=self.system_prompt,
             user_prompt=self.user_prompt.format(
                 prompt_to_test=self.inputs.model.prompt.template
             ),
         )
-        score = self.get_score(response)
-        explanation = self.get_explanation(response)
+        score = get_score(response)
+        explanation = get_explanation(response)
 
         passed = score > self.params["min_threshold"]
         results = [

validmind/tests/prompt_validation/Conciseness.py

@@ -7,6 +7,7 @@ from typing import List
 
 import pandas as pd
 
+from validmind.errors import MissingRequiredTestInputError
 from validmind.vm_models import (
     ResultSummary,
     ResultTable,
@@ -15,11 +16,16 @@ from validmind.vm_models import (
     ThresholdTestResult,
 )
 
-from .ai_powered_test import AIPoweredTest
+from .ai_powered_test import (
+    call_model,
+    get_explanation,
+    get_score,
+    missing_prompt_message,
+)
 
 
 @dataclass
-class Conciseness(ThresholdTest, AIPoweredTest):
+class Conciseness(ThresholdTest):
     """
     Analyzes and grades the conciseness of prompts provided to a Large Language Model.
 
@@ -95,12 +101,6 @@ Prompt:
 """
 '''.strip()
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)  # Call ThresholdTest.__init__
-        AIPoweredTest.__init__(
-            self, *args, **kwargs
-        )  # Explicitly call AIPoweredTest.__init__
-
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         result = results[0]
         results_table = [
@@ -124,14 +124,17 @@ Prompt:
         )
 
     def run(self):
-        response = self.call_model(
+        if not hasattr(self.inputs.model, "prompt"):
+            raise MissingRequiredTestInputError(missing_prompt_message)
+
+        response = call_model(
             system_prompt=self.system_prompt,
             user_prompt=self.user_prompt.format(
                 prompt_to_test=self.inputs.model.prompt.template
             ),
         )
-        score = self.get_score(response)
-        explanation = self.get_explanation(response)
+        score = get_score(response)
+        explanation = get_explanation(response)
 
         passed = score > self.params["min_threshold"]
         results = [

validmind/tests/prompt_validation/Delimitation.py

@@ -7,6 +7,7 @@ from typing import List
 
 import pandas as pd
 
+from validmind.errors import MissingRequiredTestInputError
 from validmind.vm_models import (
     ResultSummary,
     ResultTable,
@@ -15,11 +16,16 @@ from validmind.vm_models import (
     ThresholdTestResult,
 )
 
-from .ai_powered_test import AIPoweredTest
+from .ai_powered_test import (
+    call_model,
+    get_explanation,
+    get_score,
+    missing_prompt_message,
+)
 
 
 @dataclass
-class Delimitation(ThresholdTest, AIPoweredTest):
+class Delimitation(ThresholdTest):
     """
     Evaluates the proper use of delimiters in prompts provided to Large Language Models.
 
@@ -85,12 +91,6 @@ Prompt:
 """
 '''.strip()
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)  # Call ThresholdTest.__init__
-        AIPoweredTest.__init__(
-            self, *args, **kwargs
-        )  # Explicitly call AIPoweredTest.__init__
-
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         result = results[0]
         results_table = [
@@ -114,14 +114,17 @@ Prompt:
         )
 
     def run(self):
-        response = self.call_model(
+        if not hasattr(self.inputs.model, "prompt"):
+            raise MissingRequiredTestInputError(missing_prompt_message)
+
+        response = call_model(
             system_prompt=self.system_prompt,
             user_prompt=self.user_prompt.format(
                 prompt_to_test=self.inputs.model.prompt.template
             ),
         )
-        score = self.get_score(response)
-        explanation = self.get_explanation(response)
+        score = get_score(response)
+        explanation = get_explanation(response)
 
         passed = score > self.params["min_threshold"]
         results = [