validmind 2.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl

validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py

@@ -103,8 +103,8 @@ class RegressionModelForecastPlotLevels(Metric):
             train_ds = datasets[0]
             test_ds = datasets[1]
 
-            y_pred = train_ds.y_pred(fitted_model.input_id)
-            y_pred_test = test_ds.y_pred(fitted_model.input_id)
+            y_pred = train_ds.y_pred(fitted_model)
+            y_pred_test = test_ds.y_pred(fitted_model)
 
             all_dates = pd.concat([pd.Series(train_ds.index), pd.Series(test_ds.index)])
 

validmind/tests/model_validation/statsmodels/RegressionModelInsampleComparison.py

@@ -108,7 +108,7 @@ class RegressionModelInsampleComparison(Metric):
         for i, model in enumerate(models):
             X_columns = dataset.get_features_columns()
             y_true = dataset.y
-            y_pred = dataset.y_pred(model.model_id)
+            y_pred = dataset.y_pred(model)
 
             # Extract R-squared and Adjusted R-squared
             r2 = r2_score(y_true, y_pred)

validmind/tests/model_validation/statsmodels/RegressionModelOutsampleComparison.py

@@ -102,7 +102,7 @@ class RegressionModelOutsampleComparison(Metric):
             y_test = dataset.y
 
             # Predict the test data
-            y_pred = dataset.y_pred(fitted_model.input_id)
+            y_pred = dataset.y_pred(fitted_model)
 
             # Calculate the residuals
             residuals = y_test - y_pred

validmind/tests/model_validation/statsmodels/RegressionModelSummary.py

@@ -60,7 +60,7 @@ class RegressionModelSummary(Metric):
         X_columns = self.inputs.dataset.get_features_columns()
 
         y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(self.inputs.model.input_id)
+        y_pred = self.inputs.dataset.y_pred(self.inputs.model)
 
         r2 = r2_score(y_true, y_pred)
         adj_r2 = adj_r2_score(y_true, y_pred, len(y_true), len(X_columns))

validmind/tests/model_validation/statsmodels/RegressionModelsPerformance.py

@@ -79,10 +79,10 @@ class RegressionModelsPerformance(Metric):
     def sample_performance_ols(self, models, datasets):
         evaluation_results = []
 
-        for (model, dataset) in zip(models, datasets):
+        for model, dataset in zip(models, datasets):
             X_columns = dataset.get_features_columns()
             y_true = dataset.y
-            y_pred = dataset.y_pred(model.input_id)
+            y_pred = dataset.y_pred(model)
 
             # Extract R-squared and Adjusted R-squared
             r2 = r2_score(y_true, y_pred)

validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py

@@ -0,0 +1,128 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from dataclasses import dataclass
+
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+from sklearn.metrics import r2_score
+from sklearn.utils import check_random_state
+
+from validmind.errors import SkipTestError
+from validmind.logging import get_logger
+from validmind.vm_models import Figure, Metric
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class RegressionPermutationFeatureImportance(Metric):
+    """
+    Assesses the significance of each feature in a model by evaluating the impact on model performance when feature
+    values are randomly rearranged. Specifically designed for use with statsmodels, this metric offers insight into the
+    importance of features based on the decrease in model's predictive accuracy, typically R².
+
+    **Purpose**: The primary purpose of this metric is to determine which features significantly impact the performance
+    of a regression model developed using statsmodels. The metric measures how much the prediction accuracy deteriorates
+    when each feature's values are permuted.
+
+    **Test Mechanism**: This metric shuffles the values of each feature one at a time in the dataset, computes the model's
+    performance after each permutation, and compares it to the baseline performance. A significant decrease in performance
+    indicates the importance of the feature.
+
+    **Signs of High Risk**:
+    - Significant reliance on a feature that when permuted leads to a substantial decrease in performance, suggesting
+    overfitting or high model dependency on that feature.
+    - Features identified as unimportant despite known impacts from domain knowledge, suggesting potential issues in
+    model training or data preprocessing.
+
+    **Strengths**:
+    - Directly assesses the impact of each feature on model performance, providing clear insights into model dependencies.
+    - Model-agnostic within the scope of statsmodels, applicable to any regression model that outputs predictions.
+
+    **Limitations**:
+    - The metric is specific to statsmodels and cannot be used with other types of models without adaptation.
+    - It does not capture interactions between features, which can lead to underestimating the importance of correlated
+    features.
+    - Assumes independence of features when calculating importance, which might not always hold true.
+    """
+
+    name = "regression_pfi"
+    required_inputs = ["model", "dataset"]
+    default_params = {
+        "fontsize": 12,
+        "figure_height": 500,
+    }
+    metadata = {
+        "task_types": ["regression"],
+        "tags": [
+            "statsmodels",
+            "feature_importance",
+            "visualization",
+        ],
+    }
+
+    def run(self):
+        x = self.inputs.dataset.x_df()
+        y = self.inputs.dataset.y_df()
+
+        model = self.inputs.model.model
+        if not hasattr(model, "predict"):
+            raise SkipTestError(
+                "Model does not support 'predict' method required for PFI"
+            )
+
+        # Calculate baseline performance
+        baseline_performance = r2_score(y, model.predict(x))
+        importances = pd.DataFrame(index=x.columns, columns=["Importance", "Std Dev"])
+
+        for column in x.columns:
+            shuffled_scores = []
+            for _ in range(30):  # Default number of shuffles
+                x_shuffled = x.copy()
+                x_shuffled[column] = check_random_state(0).permutation(
+                    x_shuffled[column]
+                )
+                permuted_performance = r2_score(y, model.predict(x_shuffled))
+                shuffled_scores.append(baseline_performance - permuted_performance)
+
+            importances.loc[column] = {
+                "Importance": np.mean(shuffled_scores),
+                "Std Dev": np.std(shuffled_scores),
+            }
+
+        sorted_idx = importances["Importance"].argsort()
+
+        # Plotting the results
+        fig = go.Figure()
+        fig.add_trace(
+            go.Bar(
+                y=importances.index[sorted_idx],
+                x=importances.loc[importances.index[sorted_idx], "Importance"],
+                orientation="h",
+                error_x=dict(
+                    type="data",
+                    array=importances.loc[importances.index[sorted_idx], "Std Dev"],
+                ),
+            )
+        )
+        fig.update_layout(
+            title_text="Permutation Feature Importances",
+            yaxis=dict(
+                tickmode="linear", dtick=1, tickfont=dict(size=self.params["fontsize"])
+            ),
+            height=self.params["figure_height"],
+        )
+
+        return self.cache_results(
+            metric_value=importances.to_dict(),
+            figures=[
+                Figure(
+                    for_object=self,
+                    key="regression_pfi",
+                    figure=fig,
+                ),
+            ],
+        )

validmind/tests/model_validation/statsmodels/ScorecardHistogram.py

@@ -4,10 +4,8 @@
 
 from dataclasses import dataclass
 
-import numpy as np
-import pandas as pd
 import plotly.graph_objects as go
-from plotly.subplots import make_subplots
+from matplotlib import cm
 
 from validmind.vm_models import Figure, Metric
 
@@ -53,120 +51,89 @@ class ScorecardHistogram(Metric):
     """
 
     name = "scorecard_histogram"
-    required_inputs = ["model", "datasets"]
+    required_inputs = ["datasets"]
     metadata = {
         "task_types": ["classification"],
         "tags": ["tabular_data", "visualization", "credit_risk"],
     }
     default_params = {
         "title": "Histogram of Scores",
-        "target_score": 600,
-        "target_odds": 50,
-        "pdo": 20,
+        "score_column": "score",
     }
 
     @staticmethod
-    def compute_scores(model, X, target_score, target_odds, pdo):
-        X_copy = X.copy()
-        beta = model.model.params.values
-        alpha = model.model.params[0]
-        factor = pdo / np.log(2)
-        offset = target_score - (factor * np.log(target_odds))
-
-        for _, row in X_copy.iterrows():
-            score_i = 0
-            for i in range(1, len(beta)):
-                WoE_i = row[i]
-                score_i += (beta[i] * WoE_i) * factor
-
-            score_i += alpha * factor
-            score_i += offset
-            X_copy.loc[row.name, "score"] = score_i
-
-        return X_copy
-
-    @staticmethod
-    def plot_score_histogram(df_train, df_test, score_col, target_col, title):
-        scores_train_0 = df_train[df_train[target_col] == 0][score_col]
-        scores_train_1 = df_train[df_train[target_col] == 1][score_col]
-        scores_test_0 = df_test[df_test[target_col] == 0][score_col]
-        scores_test_1 = df_test[df_test[target_col] == 1][score_col]
-
-        fig = make_subplots(rows=1, cols=2, subplot_titles=("Train Data", "Test Data"))
-
-        trace_train_0 = go.Histogram(
-            x=scores_train_0, opacity=0.75, name=f"Train {target_col} = 0"
-        )
-        trace_train_1 = go.Histogram(
-            x=scores_train_1, opacity=0.75, name=f"Train {target_col} = 1"
-        )
-        trace_test_0 = go.Histogram(
-            x=scores_test_0, opacity=0.75, name=f"Test {target_col} = 0"
-        )
-        trace_test_1 = go.Histogram(
-            x=scores_test_1, opacity=0.75, name=f"Test {target_col} = 1"
-        )
-
-        fig.add_trace(trace_train_0, row=1, col=1)
-        fig.add_trace(trace_train_1, row=1, col=1)
-        fig.add_trace(trace_test_0, row=1, col=2)
-        fig.add_trace(trace_test_1, row=1, col=2)
-
-        fig.update_layout(barmode="overlay", title_text=title)
-
-        return fig
+    def plot_score_histogram(dataframes, dataset_titles, score_col, target_col, title):
+        figures = []
+        # Generate a colormap and convert to Plotly-accepted color format
+        # Adjust 'viridis' to any other matplotlib colormap if desired
+        colormap = cm.get_cmap("viridis")
+
+        for _, (df, dataset_title) in enumerate(zip(dataframes, dataset_titles)):
+            fig = go.Figure()
+
+            # Get unique classes and assign colors
+            classes = sorted(df[target_col].unique())
+            colors = [
+                colormap(i / len(classes))[:3] for i in range(len(classes))
+            ]  # RGB
+            color_dict = {
+                cls: f"rgb({int(rgb[0]*255)}, {int(rgb[1]*255)}, {int(rgb[2]*255)})"
+                for cls, rgb in zip(classes, colors)
+            }
+
+            for class_value in sorted(df[target_col].unique()):
+                scores_class = df[df[target_col] == class_value][score_col]
+                fig.add_trace(
+                    go.Histogram(
+                        x=scores_class,
+                        opacity=0.75,
+                        name=f"{dataset_title} {target_col} = {class_value}",
+                        marker=dict(
+                            color=color_dict[class_value],
+                        ),
+                    )
+                )
+            fig.update_layout(
+                barmode="overlay",
+                title_text=f"{title} - {dataset_title}",
+                xaxis_title="Score",
+                yaxis_title="Frequency",
+                legend_title=target_col,
+            )
+            figures.append(fig)
+        return figures
 
     def run(self):
-        model = (
-            self.inputs.model[0]
-            if isinstance(self.inputs.model, list)
-            else self.inputs.model
-        )
-
-        target_column = model.train_ds.target_column
         title = self.params["title"]
-        target_score = self.params["target_score"]
-        target_odds = self.params["target_odds"]
-        pdo = self.params["pdo"]
-
-        # Create a copy of training and testing dataframes
-        df_train = self.inputs.datasets[0].df.copy()
-        df_test = self.inputs.datasets[1].df.copy()
-
-        # Drop target_column to create feature dataframes
-        X_train = df_train.drop(columns=[target_column])
-        X_test = df_test.drop(columns=[target_column])
+        score_column = self.params["score_column"]
+        dataset_titles = [dataset.input_id for dataset in self.inputs.datasets]
+        target_column = self.inputs.datasets[0].target_column
+
+        dataframes = []
+        metric_value = {"score_histogram": {}}
+        for dataset in self.inputs.datasets:
+            df = dataset.df.copy()
+            # Check if the score_column exists in the DataFrame
+            if score_column not in df.columns:
+                raise ValueError(
+                    f"The required column '{score_column}' is not present in the dataset with input_id {dataset.input_id}"
+                )
 
-        # Subset only target_column to create target dataframes
-        y_train = df_train[[target_column]]
-        y_test = df_test[[target_column]]
+            df[score_column] = dataset.get_extra_column(score_column)
+            dataframes.append(df)
+            metric_value["score_histogram"][dataset.input_id] = list(df[score_column])
 
-        X_train_scores = self.compute_scores(
-            model, X_train, target_score, target_odds, pdo
-        )
-        X_test_scores = self.compute_scores(
-            model, X_test, target_score, target_odds, pdo
+        figures = self.plot_score_histogram(
+            dataframes, dataset_titles, score_column, target_column, title
         )
 
-        df_train = pd.concat([X_train_scores, y_train], axis=1)
-        df_test = pd.concat([X_test_scores, y_test], axis=1)
-
-        fig = self.plot_score_histogram(
-            df_train, df_test, "score", target_column, title
-        )
+        figures_list = [
+            Figure(
+                for_object=self,
+                key=f"score_histogram_{title.replace(' ', '_')}_{i+1}",
+                figure=fig,
+            )
+            for i, fig in enumerate(figures)
+        ]
 
-        return self.cache_results(
-            metric_value={
-                "score_histogram": {
-                    "train_scores": list(X_train_scores["score"]),
-                    "test_scores": list(X_test_scores["score"]),
-                },
-            },
-            figures=[
-                Figure(
-                    for_object=self,
-                    key="score_histogram",
-                    figure=fig,
-                )
-            ],
-        )
+        return self.cache_results(metric_value=metric_value, figures=figures_list)

validmind/tests/prompt_validation/ai_powered_test.py

@@ -57,6 +57,8 @@ class AIPoweredTest:
                     {"role": "system", "content": system_prompt},
                     {"role": "user", "content": user_prompt},
                 ],
+                temperature=0.0,
+                seed=42,
             )
             .choices[0]
             .message.content

validmind/tests/test_providers.py

@@ -5,44 +5,30 @@
 import importlib.util
 import os
 import sys
-
-import requests
+from typing import Protocol
 
 from validmind.logging import get_logger
 
 logger = get_logger(__name__)
 
 
-class GithubTestProviderDownloadError(Exception):
-    """
-    When the remote file can't be downloaded from the repo.
-    """
-
-    pass
-
+class TestProvider(Protocol):
+    """Protocol for user-defined test providers"""
 
-class GithubTestProviderWriteFileError(Exception):
-    """
-    When the remote file can't be downloaded from the repo.
-    """
-
-    pass
-
-
-class GithubTestProviderLoadModuleError(Exception):
-    """
-    When the remote file was downloaded but the module can't be loaded.
-    """
-
-    pass
+    def load_test(self, test_id: str):
+        """Load the test by test ID
 
+        Args:
+            test_id (str): The test ID (does not contain the namespace under which
+                the test is registered)
 
-class GithubTestProviderLoadTestError(Exception):
-    """
-    When the module was loaded but the test class can't be located.
-    """
+        Returns:
+            Test: A test class or function
 
-    pass
+        Raises:
+            FileNotFoundError: If the test is not found
+        """
+        ...
 
 
 class LocalTestProviderLoadModuleError(Exception):
@@ -61,102 +47,6 @@ class LocalTestProviderLoadTestError(Exception):
     pass
 
 
-class GithubTestProvider:
-    """
-    A class used to download python files from a Github repository and
-    dynamically load and execute the tests from those files.
-    """
-
-    BASE_URL = "https://api.github.com/repos"
-
-    def __init__(self, org: str, repo: str, token: str):
-        """
-        Initialize the GithubTestProvider with the given org, repo, and token.
-
-        Args:
-            org (str): The Github organization.
-            repo (str): The Github repository.
-            token (str): The Github access token.
-        """
-        self.org = org
-        self.repo = repo
-        self.token = token
-
-    def _download_file(self, test_path: str) -> str:
-        """
-        Download the file at the given test_path from the Github repository.
-
-        Args:
-            test_path (str): The path of the file in the repository.
-
-        Returns:
-            str: The local file path where the file was downloaded.
-
-        Raises:
-            Exception: If the file can't be downloaded or written.
-        """
-        url = f"{self.BASE_URL}/{self.org}/{self.repo}/contents/{test_path}"
-
-        headers = {
-            "Authorization": f"token {self.token}",
-            "Accept": "application/vnd.github.v3.raw",
-            "X-Github-Api-Version": "2022-11-28",
-        }
-
-        try:
-            response = requests.get(url, headers=headers)
-            response.raise_for_status()
-        except requests.RequestException as e:
-            raise GithubTestProviderDownloadError(
-                f"Failed to download the file at {url}. Error: {str(e)}"
-            )
-
-        file_path = f"/tmp/{os.path.basename(test_path)}"
-        try:
-            with open(file_path, "w") as file:
-                file.write(response.text)
-        except IOError as e:
-            raise GithubTestProviderWriteFileError(
-                f"Failed to write the file to {file_path}. Error: {str(e)}"
-            )
-
-        return file_path
-
-    def load_test(self, test_id):
-        """
-        Load the test identified by the given test_id.
-
-        Args:
-            test_id (str): The identifier of the test. This corresponds to the
-            relative path of the python file in the repository, with slashes replaced by dots.
-
-        Returns:
-            The test class that matches the last part of the test_id.
-
-        Raises:
-            Exception: If the test can't be imported or loaded.
-        """
-        test_path = f"{test_id.replace('.', '/')}.py"
-        file_path = self._download_file(test_path)
-
-        try:
-            spec = importlib.util.spec_from_file_location(test_id, file_path)
-            module = importlib.util.module_from_spec(spec)
-            spec.loader.exec_module(module)
-        except Exception as e:
-            raise GithubTestProviderLoadModuleError(
-                f"Failed to load the module from {file_path}. Error: {str(e)}"
-            )
-
-        try:
-            # find the test class that matches the last part of the test_id
-            return getattr(module, test_id.split(".")[-1])
-        except AttributeError as e:
-            raise GithubTestProviderLoadTestError(
-                f"Failed to find the test class in the module. Error: {str(e)}"
-            )
-
-
 class LocalTestProvider:
     """
     Test providers in ValidMind are responsible for loading tests from different sources,