validmind 2.2.5__py3-none-any.whl → 2.3.1__py3-none-any.whl

validmind/logging.py

@@ -13,22 +13,45 @@ from sentry_sdk.utils import event_from_exception, exc_info_from_error
 
 from .__version__ import __version__
 
-__log_level = None
 __dsn = "https://48f446843657444aa1e2c0d716ef864b@o1241367.ingest.sentry.io/4505239625465856"
 
 
 def _get_log_level():
-    """Get the log level from the environment variable if not already set"""
-    if __log_level is not None:
-        return __log_level
+    """Get the log level from the environment variable"""
+    log_level_str = os.getenv("LOG_LEVEL", "INFO").upper()
 
-    log_level_str = os.environ.get("LOG_LEVEL", "INFO").upper()
     if log_level_str not in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]:
         raise ValueError(f"Invalid log level: {log_level_str}")
 
     return logging.getLevelName(log_level_str)
 
 
+def get_logger(name="validmind", log_level=None):
+    """Get a logger for the given module name"""
+    formatter = logging.Formatter(
+        fmt="%(asctime)s - %(levelname)s(%(name)s): %(message)s"
+    )
+
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+
+    logger = logging.getLogger(name)
+    logger.setLevel(log_level or _get_log_level())
+
+    # Clear existing handlers if any (or refine the existing logic as necessary)
+    # TODO: move this to a yaml config and only configure once
+    if not any(
+        isinstance(h, type(handler)) and h.formatter._fmt == formatter._fmt
+        for h in logger.handlers
+    ):
+        logger.addHandler(handler)
+
+    # Prevent logger from propagating to root logger
+    logger.propagate = False
+
+    return logger
+
+
 def init_sentry(server_config):
     """Initialize Sentry SDK for sending logs back to ValidMind
 
@@ -42,7 +65,10 @@ def init_sentry(server_config):
             - dsn (str): The Sentry DSN
             ...: Other config options for Sentry
     """
-    if server_config.get("send_logs", False) is False:
+    if os.getenv("VM_NO_TELEMETRY", False):
+        return
+
+    if not server_config.get("send_logs", False):
         return
 
     config = {
@@ -53,33 +79,13 @@ def init_sentry(server_config):
         "environment": "production",
     }
     config.update({k: v for k, v in server_config.items() if k != "send_logs"})
-    sentry_sdk.init(**config)
-
-
-def get_logger(name="validmind", log_level=None):
-    """Get a logger for the given name"""
-    formatter = logging.Formatter(
-        fmt="%(asctime)s - %(levelname)s(%(name)s): %(message)s"
-    )
-
-    handler = logging.StreamHandler()
-    handler.setFormatter(formatter)
-
-    logger = logging.getLogger(name)
-    logger.setLevel(log_level or _get_log_level())
-
-    # Clear existing handlers if any (or refine the existing logic as necessary)
-    # TODO: lets add some better handler management
-    if not any(
-        isinstance(h, type(handler)) and h.formatter._fmt == formatter._fmt
-        for h in logger.handlers
-    ):
-        logger.addHandler(handler)
-
-    # Prevent logger from propagating to root logger
-    logger.propagate = False
 
-    return logger
+    try:
+        sentry_sdk.init(**config)
+    except Exception as e:
+        logger = get_logger(__name__)
+        logger.info("Sentry failed to initialize - ignoring...")
+        logger.debug(f"Sentry error: {str(e)}")
 
 
 def log_performance(func, name=None, logger=None, force=False):

validmind/models/foundation.py

@@ -15,7 +15,7 @@ logger = get_logger(__name__)
 @dataclass
 class Prompt:
     template: str
-    variables: list
+    variables: list = None
 
 
 class FoundationModel(FunctionModel):
@@ -33,17 +33,21 @@ class FoundationModel(FunctionModel):
     """
 
     def __post_init__(self):
-        if not getattr(self, "predict_fn") or not callable(self.predict_fn):
-            raise ValueError("FoundationModel requires a callable predict_fn")
+        super().__post_init__()
 
-        self.name = self.name or self.predict_fn.__name__
+        if not hasattr(self, "prompt") or not isinstance(self.prompt, Prompt):
+            raise ValueError("FoundationModel requires a Prompt object")
 
     def _build_prompt(self, x: pd.DataFrame):
         """
         Builds the prompt for the model
         """
-        return self.prompt.template.format(
-            **{key: x[key] for key in self.prompt.variables}
+        return (
+            self.prompt.template.format(
+                **{key: x[key] for key in self.prompt.variables}
+            )
+            if self.prompt.variables
+            else self.prompt.template
         )
 
     def predict(self, X: pd.DataFrame):

validmind/models/function.py

@@ -31,10 +31,12 @@ class FunctionModel(VMModel):
             input features and return a prediction.
         input_id (str, optional): The input ID for the model. Defaults to None.
         name (str, optional): The name of the model. Defaults to the name of the predict_fn.
+        prompt (Prompt, optional): If using a prompt, the prompt object that defines the template
+            and the variables (if any). Defaults to None.
     """
 
     def __post_init__(self):
-        if not getattr(self, "predict_fn") or not callable(self.predict_fn):
+        if not hasattr(self, "predict_fn") or not callable(self.predict_fn):
             raise ValueError("FunctionModel requires a callable predict_fn")
 
         self.name = self.name or self.predict_fn.__name__

validmind/models/metadata.py

@@ -24,7 +24,7 @@ class MetadataModel(VMModel):
     """
 
     def __post_init__(self):
-        if not getattr(self, "attributes"):
+        if not hasattr(self, "attributes"):
             raise ValueError("MetadataModel requires attributes")
 
         self.name = self.name or "Metadata Model"

validmind/test_suites/__init__.py

@@ -25,12 +25,7 @@ from .embeddings import EmbeddingsFullSuite, EmbeddingsMetrics, EmbeddingsPerfor
 from .llm import LLMClassifierFullSuite, PromptValidation
 from .nlp import NLPClassifierFullSuite
 from .parameters_optimization import KmeansParametersOptimization
-from .regression import (
-    RegressionFullSuite,
-    RegressionMetrics,
-    RegressionModelsComparison,
-    RegressionPerformance,
-)
+from .regression import RegressionFullSuite, RegressionMetrics, RegressionPerformance
 from .statsmodels_timeseries import (
     RegressionModelDescription,
     RegressionModelsEvaluation,
@@ -72,7 +67,6 @@ core_test_suites = {
     RegressionMetrics.suite_id: RegressionMetrics,
     RegressionModelDescription.suite_id: RegressionModelDescription,
     RegressionModelsEvaluation.suite_id: RegressionModelsEvaluation,
-    RegressionModelsComparison.suite_id: RegressionModelsComparison,
     RegressionFullSuite.suite_id: RegressionFullSuite,
     RegressionPerformance.suite_id: RegressionPerformance,
     SummarizationMetrics.suite_id: SummarizationMetrics,

validmind/test_suites/regression.py

@@ -32,17 +32,6 @@ class RegressionPerformance(TestSuite):
     ]
 
 
-class RegressionModelsComparison(TestSuite):
-    """
-    Test suite for regression models performance comparison
-    """
-
-    suite_id = "regression_models_comparison"
-    tests = [
-        "validmind.model_validation.sklearn.RegressionModelsPerformanceComparison",
-    ]
-
-
 class RegressionFullSuite(TestSuite):
     """
     Full test suite for regression models.
@@ -70,9 +59,4 @@ class RegressionFullSuite(TestSuite):
             "section_description": RegressionPerformance.__doc__,
             "section_tests": RegressionPerformance.tests,
         },
-        {
-            "section_id": RegressionModelsComparison.suite_id,
-            "section_description": RegressionModelsComparison.__doc__,
-            "section_tests": RegressionModelsComparison.tests,
-        },
     ]

validmind/test_suites/statsmodels_timeseries.py

@@ -29,5 +29,5 @@ class RegressionModelsEvaluation(TestSuite):
     suite_id = "regression_models_evaluation"
     tests = [
         "validmind.model_validation.statsmodels.RegressionModelsCoeffs",
-        "validmind.model_validation.statsmodels.RegressionModelsPerformance",
+        "validmind.model_validation.sklearn.RegressionModelsPerformanceComparison",
     ]

validmind/tests/data_validation/ACFandPACFPlot.py

@@ -2,9 +2,9 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-import matplotlib.pyplot as plt
 import pandas as pd
-from statsmodels.graphics.tsaplots import plot_acf, plot_pacf
+import plotly.graph_objects as go
+from statsmodels.tsa.stattools import acf, pacf
 
 from validmind.vm_models import Figure, Metric
 
@@ -77,37 +77,46 @@ class ACFandPACFPlot(Metric):
         for col in df.columns:
             series = df[col]
 
-            # Create subplots
-            fig, (ax1, ax2) = plt.subplots(1, 2)
-            width, _ = fig.get_size_inches()
-            fig.set_size_inches(width, 5)
-
-            plot_acf(series, ax=ax1)
-            plot_pacf(series, ax=ax2)
-
-            # Get the current y-axis limits
-            ymin, ymax = ax1.get_ylim()
-            # Set new limits - adding a bit of space
-            ax1.set_ylim([ymin, ymax + 0.05 * (ymax - ymin)])
+            # Calculate the maximum number of lags based on the size of the dataset
+            max_lags = min(40, len(series) // 2 - 1)
+
+            # Calculate ACF and PACF values
+            acf_values = acf(series, nlags=max_lags)
+            pacf_values = pacf(series, nlags=max_lags)
+
+            # Create ACF plot using Plotly
+            acf_fig = go.Figure()
+            acf_fig.add_trace(go.Bar(x=list(range(len(acf_values))), y=acf_values))
+            acf_fig.update_layout(
+                title=f"ACF for {col}",
+                xaxis_title="Lag",
+                yaxis_title="ACF",
+                font=dict(size=18),
+            )
 
-            ymin, ymax = ax2.get_ylim()
-            ax2.set_ylim([ymin, ymax + 0.05 * (ymax - ymin)])
+            # Create PACF plot using Plotly
+            pacf_fig = go.Figure()
+            pacf_fig.add_trace(go.Bar(x=list(range(len(pacf_values))), y=pacf_values))
+            pacf_fig.update_layout(
+                title=f"PACF for {col}",
+                xaxis_title="Lag",
+                yaxis_title="PACF",
+                font=dict(size=18),
+            )
 
-            ax1.tick_params(axis="both", labelsize=18)
-            ax2.tick_params(axis="both", labelsize=18)
-            ax1.set_title(f"ACF for {col}", weight="bold", fontsize=20)
-            ax2.set_title(f"PACF for {col}", weight="bold", fontsize=20)
-            ax1.set_xlabel("Lag", fontsize=18)
-            ax2.set_xlabel("Lag", fontsize=18)
             figures.append(
                 Figure(
                     for_object=self,
-                    key=f"{self.key}:{col}",
-                    figure=fig,
+                    key=f"{self.key}:{col}_acf",
+                    figure=acf_fig,
+                )
+            )
+            figures.append(
+                Figure(
+                    for_object=self,
+                    key=f"{self.key}:{col}_pacf",
+                    figure=pacf_fig,
                 )
             )
-
-            # Do this if you want to prevent the figure from being displayed
-            plt.close("all")
 
         return self.cache_results(figures=figures)

validmind/tests/{model_validation/statsmodels → data_validation}/ADF.py

@@ -2,12 +2,18 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from pandas import DataFrame
+from dataclasses import dataclass
+
+import pandas as pd
 from statsmodels.tsa.stattools import adfuller
 
+from validmind.logging import get_logger
 from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
 
+logger = get_logger(__name__)
+
 
+@dataclass
 class ADF(Metric):
     """
     Assesses the stationarity of a time series dataset using the Augmented Dickey-Fuller (ADF) test.
@@ -53,7 +59,7 @@ class ADF(Metric):
     }
 
     def summary(self, metric_value: dict):
-        table = DataFrame.from_dict(metric_value, orient="index")
+        table = pd.DataFrame.from_dict(metric_value, orient="index")
         table = table.reset_index()
         table.columns = [
             "Feature",
@@ -83,18 +89,41 @@ class ADF(Metric):
         """
         dataset = self.inputs.dataset.df
 
+        # Check if the dataset is a time series
+        if not isinstance(dataset.index, (pd.DatetimeIndex, pd.PeriodIndex)):
+            raise ValueError(
+                "Dataset index must be a datetime or period index for time series analysis."
+            )
+
+        # Preprocessing: Drop rows with any NaN values
+        if dataset.isnull().values.any():
+            logger.warning(
+                "Dataset contains missing values. Rows with NaNs will be dropped."
+            )
+            dataset = dataset.dropna()
+
         adf_values = {}
         for col in dataset.columns:
-            adf, pvalue, usedlag, nobs, critical_values, icbest = adfuller(
-                dataset[col].values
-            )
-            adf_values[col] = {
-                "stat": adf,
-                "pvalue": pvalue,
-                "usedlag": usedlag,
-                "nobs": nobs,
-                "critical_values": critical_values,
-                "icbest": icbest,
-            }
+            try:
+                adf_result = adfuller(dataset[col].values)
+                adf_values[col] = {
+                    "ADF Statistic": adf_result[0],
+                    "P-Value": adf_result[1],
+                    "Used Lag": adf_result[2],
+                    "Number of Observations": adf_result[3],
+                    "Critical Values": adf_result[4],
+                    "IC Best": adf_result[5],
+                }
+            except Exception as e:
+                logger.error(f"Error processing column '{col}': {e}")
+                adf_values[col] = {
+                    "ADF Statistic": None,
+                    "P-Value": None,
+                    "Used Lag": None,
+                    "Number of Observations": None,
+                    "Critical Values": None,
+                    "IC Best": None,
+                    "Error": str(e),
+                }
 
         return self.cache_results(adf_values)

validmind/tests/data_validation/BivariateScatterPlots.py

@@ -2,10 +2,10 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import itertools
 from dataclasses import dataclass
 
-import matplotlib.pyplot as plt
-import seaborn as sns
+import plotly.express as px
 
 from validmind.vm_models import Figure, Metric
 
@@ -23,7 +23,7 @@ class BivariateScatterPlots(Metric):
     biases and irregularities in the data.
 
     **Test Mechanism**: This metric operates by creating a scatter plot for each pair of the selected features in the
-    dataset. If the parameters "features_pairs" are not specified, an error will be thrown. The metric offers
+    dataset. If the parameters "selected_columns" are not specified, an error will be thrown. The metric offers
     flexibility by allowing the user to filter on a specific target class - specified by the "target_filter" parameter
     - for more granified insights. Each scatterplot is then color-coded based on the category of the target variable
     for better visual differentiation. The seaborn scatterplot library is used for generating the plots.
@@ -53,7 +53,7 @@ class BivariateScatterPlots(Metric):
 
     name = "bivariate_scatter_plots"
     required_inputs = ["dataset"]
-    default_params = {"features_pairs": None, "target_filter": None}
+    default_params = {"selected_columns": None}
     metadata = {
         "task_types": ["classification"],
         "tags": [
@@ -65,52 +65,49 @@ class BivariateScatterPlots(Metric):
         ],
     }
 
-    def plot_bivariate_scatter(self, features_pairs, target_filter):
-        status_var = self.inputs.dataset.target_column
+    def plot_bivariate_scatter(self, columns):
         figures = []
-        for x, y in features_pairs.items():
-            df = self.inputs.dataset.df
-            if target_filter is not None:
-                df = df[df[status_var] == target_filter]
-
-            plt.figure()
-
-            # Scatterplot using seaborn, with color variation based on 'status_var'
-            # Create color mapping with rgba values, last value is alpha (transparency)
-            palette = {0: (0.8, 0.8, 0.8, 0.8), 1: "tab:red"}
-            plot = sns.scatterplot(
-                data=df, x=x, y=y, hue=status_var, palette=palette, alpha=1
+        df = self.inputs.dataset.df
+
+        # Generate all pairs of columns
+        features_pairs = list(itertools.combinations(columns, 2))
+
+        for x, y in features_pairs:
+            fig = px.scatter(
+                df,
+                x=x,
+                y=y,
+                title=f"{x} and {y}",
+                labels={x: x, y: y},
+                opacity=0.7,
+                color_discrete_sequence=["blue"],  # Use the same color for all points
             )
-
-            # Change legend labels
-            legend_labels = [
-                "Category 1" if t.get_text() == "1" else "Category 2"
-                for t in plot.legend_.texts[1:]
-            ]
-            plot.legend_.texts[1:] = legend_labels
-
-            plt.title(x + " and " + y)
-            plt.xlabel(x)
-            plt.ylabel(y)
-            plt.show()
+            fig.update_traces(marker=dict(color="blue"))
 
             figures.append(
-                Figure(for_object=self, key=f"{self.key}:{x}_{y}", figure=plt.figure())
+                Figure(for_object=self, key=f"{self.key}:{x}_{y}", figure=fig)
             )
 
-        plt.close("all")
-
         return figures
 
     def run(self):
-        features_pairs = self.params["features_pairs"]
-        target_filter = self.params["target_filter"]
-
-        if features_pairs is None:
-            raise ValueError(
-                "The features_pairs parameter is required for this metric."
-            )
+        selected_columns = self.params["selected_columns"]
+
+        if selected_columns is None:
+            # Use all columns if selected_columns is not provided
+            selected_columns = self.inputs.dataset.df.columns.tolist()
+        else:
+            # Check if all selected columns exist in the dataframe
+            missing_columns = [
+                col
+                for col in selected_columns
+                if col not in self.inputs.dataset.df.columns
+            ]
+            if missing_columns:
+                raise ValueError(
+                    f"The following selected columns are not in the dataframe: {missing_columns}"
+                )
 
-        figures = self.plot_bivariate_scatter(features_pairs, target_filter)
+        figures = self.plot_bivariate_scatter(selected_columns)
 
         return self.cache_results(figures=figures)

validmind/tests/{model_validation/statsmodels → data_validation}/DFGLSArch.py

@@ -4,9 +4,14 @@
 
 from dataclasses import dataclass
 
+import pandas as pd
 from arch.unitroot import DFGLS
+from numpy.linalg import LinAlgError
 
-from validmind.vm_models import Metric
+from validmind.logging import get_logger
+from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
+
+logger = get_logger(__name__)
 
 
 @dataclass
@@ -59,14 +64,65 @@ class DFGLSArch(Metric):
         """
         dataset = self.inputs.dataset.df
 
-        dfgls_values = {}
+        # Check if the dataset is a time series
+        if not isinstance(dataset.index, (pd.DatetimeIndex, pd.PeriodIndex)):
+            raise ValueError(
+                "Dataset index must be a datetime or period index for time series analysis."
+            )
+
+        # Preprocessing: Drop rows with any NaN values
+        if dataset.isnull().values.any():
+            logger.warning(
+                "Dataset contains missing values. Rows with NaNs will be dropped."
+            )
+            dataset = dataset.dropna()
+
+        # Convert to numeric and handle non-numeric data
+        dataset = dataset.apply(pd.to_numeric, errors="coerce")
+
+        # Initialize a list to store DFGLS results
+        dfgls_values = []
+
         for col in dataset.columns:
-            dfgls_out = DFGLS(dataset[col].values)
-            dfgls_values[col] = {
-                "stat": dfgls_out.stat,
-                "pvalue": dfgls_out.pvalue,
-                "usedlag": dfgls_out.lags,
-                "nobs": dfgls_out.nobs,
-            }
-
-        return self.cache_results(dfgls_values)
+            try:
+                dfgls_out = DFGLS(dataset[col].values)
+                dfgls_values.append(
+                    {
+                        "Variable": col,
+                        "stat": dfgls_out.stat,
+                        "pvalue": dfgls_out.pvalue,
+                        "usedlag": dfgls_out.lags,
+                        "nobs": dfgls_out.nobs,
+                    }
+                )
+            except LinAlgError as e:
+                logger.error(
+                    f"SVD did not converge while processing column '{col}'. This could be due to numerical instability or multicollinearity. Error details: {e}"
+                )
+                dfgls_values.append(
+                    {
+                        "Variable": col,
+                        "stat": None,
+                        "pvalue": None,
+                        "usedlag": None,
+                        "nobs": None,
+                        "error": str(e),
+                    }
+                )
+
+        return self.cache_results({"dfgls_results": dfgls_values})
+
+    def summary(self, metric_value):
+        """
+        Build a table for summarizing the DFGLS results
+        """
+        dfgls_results = metric_value["dfgls_results"]
+
+        return ResultSummary(
+            results=[
+                ResultTable(
+                    data=dfgls_results,
+                    metadata=ResultTableMetadata(title="DFGLS Test Results"),
+                )
+            ]
+        )

validmind/tests/data_validation/HeatmapFeatureCorrelations.py

@@ -62,7 +62,7 @@ class HeatmapFeatureCorrelations(Metric):
     }
 
     def run(self):
-        features = self.params["features"]
+        features = self.params.get("features")
         declutter = self.params.get("declutter", False)
         fontsize = self.params.get("fontsize", 13)
 

validmind/tests/data_validation/HighPearsonCorrelation.py

@@ -65,9 +65,18 @@ class HighPearsonCorrelation(ThresholdTest):
     }
 
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
-        """
-        The high pearson correlation test returns results like these:
-        [{"values": {"correlations": [{"column": "NumOfProducts", "correlation": -0.3044645622389459}]}, "column": "Balance", "passed": false}]
+        """The high pearson correlation test returns results like these:
+        [
+            {
+                "values": {
+                    "correlations": [
+                        {"column": "NumOfProducts", "correlation": -0.3044645622389459}
+                    ]
+                },
+                "column": "Balance",
+                "passed": false,
+            }
+        ]
         """
         results_table = [
             {

validmind/tests/data_validation/IsolationForestOutliers.py

@@ -64,7 +64,7 @@ class IsolationForestOutliers(Metric):
 
     def run(self):
         if self.params["features_columns"] is None:
-            features_list = self.inputs.dataset.feature_columns
+            features_list = self.inputs.dataset.feature_columns_numeric
         else:
             features_list = self.params["features_columns"]
 
@@ -78,7 +78,7 @@ class IsolationForestOutliers(Metric):
                 + "training dataset feature columns"
             )
 
-        dataset = self.inputs.dataset.df
+        dataset = self.inputs.dataset.df[features_list]
 
         # Training with isolation forest algorithm
         clf = IsolationForest(