validmind 2.0.7__py3-none-any.whl → 2.1.0__py3-none-any.whl

validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py

@@ -0,0 +1,128 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from dataclasses import dataclass
+
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+from sklearn.metrics import r2_score
+from sklearn.utils import check_random_state
+
+from validmind.errors import SkipTestError
+from validmind.logging import get_logger
+from validmind.vm_models import Figure, Metric
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class RegressionPermutationFeatureImportance(Metric):
+    """
+    Assesses the significance of each feature in a model by evaluating the impact on model performance when feature
+    values are randomly rearranged. Specifically designed for use with statsmodels, this metric offers insight into the
+    importance of features based on the decrease in model's predictive accuracy, typically R².
+
+    **Purpose**: The primary purpose of this metric is to determine which features significantly impact the performance
+    of a regression model developed using statsmodels. The metric measures how much the prediction accuracy deteriorates
+    when each feature's values are permuted.
+
+    **Test Mechanism**: This metric shuffles the values of each feature one at a time in the dataset, computes the model's
+    performance after each permutation, and compares it to the baseline performance. A significant decrease in performance
+    indicates the importance of the feature.
+
+    **Signs of High Risk**:
+    - Significant reliance on a feature that when permuted leads to a substantial decrease in performance, suggesting
+    overfitting or high model dependency on that feature.
+    - Features identified as unimportant despite known impacts from domain knowledge, suggesting potential issues in
+    model training or data preprocessing.
+
+    **Strengths**:
+    - Directly assesses the impact of each feature on model performance, providing clear insights into model dependencies.
+    - Model-agnostic within the scope of statsmodels, applicable to any regression model that outputs predictions.
+
+    **Limitations**:
+    - The metric is specific to statsmodels and cannot be used with other types of models without adaptation.
+    - It does not capture interactions between features, which can lead to underestimating the importance of correlated
+    features.
+    - Assumes independence of features when calculating importance, which might not always hold true.
+    """
+
+    name = "regression_pfi"
+    required_inputs = ["model", "dataset"]
+    default_params = {
+        "fontsize": 12,
+        "figure_height": 500,
+    }
+    metadata = {
+        "task_types": ["regression"],
+        "tags": [
+            "statsmodels",
+            "feature_importance",
+            "visualization",
+        ],
+    }
+
+    def run(self):
+        x = self.inputs.dataset.x_df()
+        y = self.inputs.dataset.y_df()
+
+        model = self.inputs.model.model
+        if not hasattr(model, "predict"):
+            raise SkipTestError(
+                "Model does not support 'predict' method required for PFI"
+            )
+
+        # Calculate baseline performance
+        baseline_performance = r2_score(y, model.predict(x))
+        importances = pd.DataFrame(index=x.columns, columns=["Importance", "Std Dev"])
+
+        for column in x.columns:
+            shuffled_scores = []
+            for _ in range(30):  # Default number of shuffles
+                x_shuffled = x.copy()
+                x_shuffled[column] = check_random_state(0).permutation(
+                    x_shuffled[column]
+                )
+                permuted_performance = r2_score(y, model.predict(x_shuffled))
+                shuffled_scores.append(baseline_performance - permuted_performance)
+
+            importances.loc[column] = {
+                "Importance": np.mean(shuffled_scores),
+                "Std Dev": np.std(shuffled_scores),
+            }
+
+        sorted_idx = importances["Importance"].argsort()
+
+        # Plotting the results
+        fig = go.Figure()
+        fig.add_trace(
+            go.Bar(
+                y=importances.index[sorted_idx],
+                x=importances.loc[importances.index[sorted_idx], "Importance"],
+                orientation="h",
+                error_x=dict(
+                    type="data",
+                    array=importances.loc[importances.index[sorted_idx], "Std Dev"],
+                ),
+            )
+        )
+        fig.update_layout(
+            title_text="Permutation Feature Importances",
+            yaxis=dict(
+                tickmode="linear", dtick=1, tickfont=dict(size=self.params["fontsize"])
+            ),
+            height=self.params["figure_height"],
+        )
+
+        return self.cache_results(
+            metric_value=importances.to_dict(),
+            figures=[
+                Figure(
+                    for_object=self,
+                    key="regression_pfi",
+                    figure=fig,
+                ),
+            ],
+        )

validmind/tests/model_validation/statsmodels/ScorecardHistogram.py

@@ -4,10 +4,8 @@
 
 from dataclasses import dataclass
 
-import numpy as np
-import pandas as pd
 import plotly.graph_objects as go
-from plotly.subplots import make_subplots
+from matplotlib import cm
 
 from validmind.vm_models import Figure, Metric
 
@@ -53,120 +51,89 @@ class ScorecardHistogram(Metric):
     """
 
     name = "scorecard_histogram"
-    required_inputs = ["model", "datasets"]
+    required_inputs = ["datasets"]
     metadata = {
         "task_types": ["classification"],
         "tags": ["tabular_data", "visualization", "credit_risk"],
     }
     default_params = {
         "title": "Histogram of Scores",
-        "target_score": 600,
-        "target_odds": 50,
-        "pdo": 20,
+        "score_column": "score",
     }
 
     @staticmethod
-    def compute_scores(model, X, target_score, target_odds, pdo):
-        X_copy = X.copy()
-        beta = model.model.params.values
-        alpha = model.model.params[0]
-        factor = pdo / np.log(2)
-        offset = target_score - (factor * np.log(target_odds))
-
-        for _, row in X_copy.iterrows():
-            score_i = 0
-            for i in range(1, len(beta)):
-                WoE_i = row[i]
-                score_i += (beta[i] * WoE_i) * factor
-
-            score_i += alpha * factor
-            score_i += offset
-            X_copy.loc[row.name, "score"] = score_i
-
-        return X_copy
-
-    @staticmethod
-    def plot_score_histogram(df_train, df_test, score_col, target_col, title):
-        scores_train_0 = df_train[df_train[target_col] == 0][score_col]
-        scores_train_1 = df_train[df_train[target_col] == 1][score_col]
-        scores_test_0 = df_test[df_test[target_col] == 0][score_col]
-        scores_test_1 = df_test[df_test[target_col] == 1][score_col]
-
-        fig = make_subplots(rows=1, cols=2, subplot_titles=("Train Data", "Test Data"))
-
-        trace_train_0 = go.Histogram(
-            x=scores_train_0, opacity=0.75, name=f"Train {target_col} = 0"
-        )
-        trace_train_1 = go.Histogram(
-            x=scores_train_1, opacity=0.75, name=f"Train {target_col} = 1"
-        )
-        trace_test_0 = go.Histogram(
-            x=scores_test_0, opacity=0.75, name=f"Test {target_col} = 0"
-        )
-        trace_test_1 = go.Histogram(
-            x=scores_test_1, opacity=0.75, name=f"Test {target_col} = 1"
-        )
-
-        fig.add_trace(trace_train_0, row=1, col=1)
-        fig.add_trace(trace_train_1, row=1, col=1)
-        fig.add_trace(trace_test_0, row=1, col=2)
-        fig.add_trace(trace_test_1, row=1, col=2)
-
-        fig.update_layout(barmode="overlay", title_text=title)
-
-        return fig
+    def plot_score_histogram(dataframes, dataset_titles, score_col, target_col, title):
+        figures = []
+        # Generate a colormap and convert to Plotly-accepted color format
+        # Adjust 'viridis' to any other matplotlib colormap if desired
+        colormap = cm.get_cmap("viridis")
+
+        for _, (df, dataset_title) in enumerate(zip(dataframes, dataset_titles)):
+            fig = go.Figure()
+
+            # Get unique classes and assign colors
+            classes = sorted(df[target_col].unique())
+            colors = [
+                colormap(i / len(classes))[:3] for i in range(len(classes))
+            ]  # RGB
+            color_dict = {
+                cls: f"rgb({int(rgb[0]*255)}, {int(rgb[1]*255)}, {int(rgb[2]*255)})"
+                for cls, rgb in zip(classes, colors)
+            }
+
+            for class_value in sorted(df[target_col].unique()):
+                scores_class = df[df[target_col] == class_value][score_col]
+                fig.add_trace(
+                    go.Histogram(
+                        x=scores_class,
+                        opacity=0.75,
+                        name=f"{dataset_title} {target_col} = {class_value}",
+                        marker=dict(
+                            color=color_dict[class_value],
+                        ),
+                    )
+                )
+            fig.update_layout(
+                barmode="overlay",
+                title_text=f"{title} - {dataset_title}",
+                xaxis_title="Score",
+                yaxis_title="Frequency",
+                legend_title=target_col,
+            )
+            figures.append(fig)
+        return figures
 
     def run(self):
-        model = (
-            self.inputs.model[0]
-            if isinstance(self.inputs.model, list)
-            else self.inputs.model
-        )
-
-        target_column = model.train_ds.target_column
         title = self.params["title"]
-        target_score = self.params["target_score"]
-        target_odds = self.params["target_odds"]
-        pdo = self.params["pdo"]
-
-        # Create a copy of training and testing dataframes
-        df_train = self.inputs.datasets[0].df.copy()
-        df_test = self.inputs.datasets[1].df.copy()
-
-        # Drop target_column to create feature dataframes
-        X_train = df_train.drop(columns=[target_column])
-        X_test = df_test.drop(columns=[target_column])
+        score_column = self.params["score_column"]
+        dataset_titles = [dataset.input_id for dataset in self.inputs.datasets]
+        target_column = self.inputs.datasets[0].target_column
+
+        dataframes = []
+        metric_value = {"score_histogram": {}}
+        for dataset in self.inputs.datasets:
+            df = dataset.df.copy()
+            # Check if the score_column exists in the DataFrame
+            if score_column not in df.columns:
+                raise ValueError(
+                    f"The required column '{score_column}' is not present in the dataset with input_id {dataset.input_id}"
+                )
 
-        # Subset only target_column to create target dataframes
-        y_train = df_train[[target_column]]
-        y_test = df_test[[target_column]]
+            df[score_column] = dataset.get_extra_column(score_column)
+            dataframes.append(df)
+            metric_value["score_histogram"][dataset.input_id] = list(df[score_column])
 
-        X_train_scores = self.compute_scores(
-            model, X_train, target_score, target_odds, pdo
-        )
-        X_test_scores = self.compute_scores(
-            model, X_test, target_score, target_odds, pdo
+        figures = self.plot_score_histogram(
+            dataframes, dataset_titles, score_column, target_column, title
         )
 
-        df_train = pd.concat([X_train_scores, y_train], axis=1)
-        df_test = pd.concat([X_test_scores, y_test], axis=1)
-
-        fig = self.plot_score_histogram(
-            df_train, df_test, "score", target_column, title
-        )
+        figures_list = [
+            Figure(
+                for_object=self,
+                key=f"score_histogram_{title.replace(' ', '_')}_{i+1}",
+                figure=fig,
+            )
+            for i, fig in enumerate(figures)
+        ]
 
-        return self.cache_results(
-            metric_value={
-                "score_histogram": {
-                    "train_scores": list(X_train_scores["score"]),
-                    "test_scores": list(X_test_scores["score"]),
-                },
-            },
-            figures=[
-                Figure(
-                    for_object=self,
-                    key="score_histogram",
-                    figure=fig,
-                )
-            ],
-        )
+        return self.cache_results(metric_value=metric_value, figures=figures_list)

validmind/tests/test_providers.py

@@ -5,44 +5,30 @@
 import importlib.util
 import os
 import sys
-
-import requests
+from typing import Protocol
 
 from validmind.logging import get_logger
 
 logger = get_logger(__name__)
 
 
-class GithubTestProviderDownloadError(Exception):
-    """
-    When the remote file can't be downloaded from the repo.
-    """
-
-    pass
-
+class TestProvider(Protocol):
+    """Protocol for user-defined test providers"""
 
-class GithubTestProviderWriteFileError(Exception):
-    """
-    When the remote file can't be downloaded from the repo.
-    """
-
-    pass
-
-
-class GithubTestProviderLoadModuleError(Exception):
-    """
-    When the remote file was downloaded but the module can't be loaded.
-    """
-
-    pass
+    def load_test(self, test_id: str):
+        """Load the test by test ID
 
+        Args:
+            test_id (str): The test ID (does not contain the namespace under which
+                the test is registered)
 
-class GithubTestProviderLoadTestError(Exception):
-    """
-    When the module was loaded but the test class can't be located.
-    """
+        Returns:
+            Test: A test class or function
 
-    pass
+        Raises:
+            FileNotFoundError: If the test is not found
+        """
+        ...
 
 
 class LocalTestProviderLoadModuleError(Exception):
@@ -61,102 +47,6 @@ class LocalTestProviderLoadTestError(Exception):
     pass
 
 
-class GithubTestProvider:
-    """
-    A class used to download python files from a Github repository and
-    dynamically load and execute the tests from those files.
-    """
-
-    BASE_URL = "https://api.github.com/repos"
-
-    def __init__(self, org: str, repo: str, token: str):
-        """
-        Initialize the GithubTestProvider with the given org, repo, and token.
-
-        Args:
-            org (str): The Github organization.
-            repo (str): The Github repository.
-            token (str): The Github access token.
-        """
-        self.org = org
-        self.repo = repo
-        self.token = token
-
-    def _download_file(self, test_path: str) -> str:
-        """
-        Download the file at the given test_path from the Github repository.
-
-        Args:
-            test_path (str): The path of the file in the repository.
-
-        Returns:
-            str: The local file path where the file was downloaded.
-
-        Raises:
-            Exception: If the file can't be downloaded or written.
-        """
-        url = f"{self.BASE_URL}/{self.org}/{self.repo}/contents/{test_path}"
-
-        headers = {
-            "Authorization": f"token {self.token}",
-            "Accept": "application/vnd.github.v3.raw",
-            "X-Github-Api-Version": "2022-11-28",
-        }
-
-        try:
-            response = requests.get(url, headers=headers)
-            response.raise_for_status()
-        except requests.RequestException as e:
-            raise GithubTestProviderDownloadError(
-                f"Failed to download the file at {url}. Error: {str(e)}"
-            )
-
-        file_path = f"/tmp/{os.path.basename(test_path)}"
-        try:
-            with open(file_path, "w") as file:
-                file.write(response.text)
-        except IOError as e:
-            raise GithubTestProviderWriteFileError(
-                f"Failed to write the file to {file_path}. Error: {str(e)}"
-            )
-
-        return file_path
-
-    def load_test(self, test_id):
-        """
-        Load the test identified by the given test_id.
-
-        Args:
-            test_id (str): The identifier of the test. This corresponds to the
-            relative path of the python file in the repository, with slashes replaced by dots.
-
-        Returns:
-            The test class that matches the last part of the test_id.
-
-        Raises:
-            Exception: If the test can't be imported or loaded.
-        """
-        test_path = f"{test_id.replace('.', '/')}.py"
-        file_path = self._download_file(test_path)
-
-        try:
-            spec = importlib.util.spec_from_file_location(test_id, file_path)
-            module = importlib.util.module_from_spec(spec)
-            spec.loader.exec_module(module)
-        except Exception as e:
-            raise GithubTestProviderLoadModuleError(
-                f"Failed to load the module from {file_path}. Error: {str(e)}"
-            )
-
-        try:
-            # find the test class that matches the last part of the test_id
-            return getattr(module, test_id.split(".")[-1])
-        except AttributeError as e:
-            raise GithubTestProviderLoadTestError(
-                f"Failed to find the test class in the module. Error: {str(e)}"
-            )
-
-
 class LocalTestProvider:
     """
     Test providers in ValidMind are responsible for loading tests from different sources,

validmind/unit_metrics/__init__.py

@@ -3,14 +3,13 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import hashlib
-import importlib
 import json
+from importlib import import_module
 
 import numpy as np
 
-from validmind.vm_models import TestInput
-
-from ..utils import get_model_info
+from ..tests.decorator import _build_result, _inspect_signature
+from ..utils import get_model_info, test_id_to_name
 
 unit_metric_results_cache = {}
 
@@ -134,52 +133,6 @@ def _fast_hash(df, sample_size=1000, model_and_prediction_info=None):
     return hash_obj.hexdigest()
 
 
-def _get_metric_class(metric_id):
-    """Get the metric class by metric_id
-
-    This function will load the metric class by metric_id.
-
-    Args:
-        metric_id (str): The full metric id (e.g. 'validmind.vm_models.test.v2.model_validation.sklearn.F1')
-
-    Returns:
-        Metric: The metric class
-    """
-
-    metric_module = importlib.import_module(f"{metric_id}")
-
-    class_name = metric_id.split(".")[-1]
-
-    # Access the class within the F1 module
-    metric_class = getattr(metric_module, class_name)
-
-    return metric_class
-
-
-def get_input_type(input_obj):
-    """
-    Determines whether the input object is a 'dataset' or 'model' based on its class module path.
-
-    Args:
-        input_obj: The object to type check.
-
-    Returns:
-        str: 'dataset' or 'model' depending on the object's module, or raises ValueError.
-    """
-    # Obtain the class object of input_obj (for clarity and debugging)
-    class_obj = input_obj.__class__
-
-    # Obtain the module name as a string from the class object
-    class_module = class_obj.__module__
-
-    if "validmind.vm_models.dataset" in class_module:
-        return "dataset"
-    elif "validmind.models" in class_module:
-        return "model"
-    else:
-        raise ValueError("Input must be of type validmind Dataset or Model")
-
-
 def get_metric_cache_key(metric_id, params, inputs):
     cache_elements = [metric_id]
 
@@ -209,34 +162,88 @@ def get_metric_cache_key(metric_id, params, inputs):
     return key
 
 
-def run_metric(metric_id=None, inputs=None, params=None):
-    """Run a single metric
-
-    This function provides a high level interface for running a single metric. A metric
-    is a single test that calculates a value based on the input data.
+def load_metric(metric_id):
+    """Load a metric class from a string
 
     Args:
-        metric_id (str): The metric name (e.g. 'F1')
-        params (dict): A dictionary of the metric parameters
+        metric_id (str): The metric id (e.g. 'validmind.unit_metrics.classification.sklearn.F1')
 
     Returns:
-        MetricResult: The metric result object
+        callable: The metric function
     """
-    cache_key = get_metric_cache_key(metric_id, params, inputs)
+    return getattr(import_module(metric_id), metric_id.split(".")[-1])
 
-    # Check if the metric value already exists in the global variable
-    if cache_key in unit_metric_results_cache:
-        return unit_metric_results_cache[cache_key]
 
-    # Load the metric class by metric_id
-    metric_class = _get_metric_class(metric_id)
+def run_metric(metric_id, inputs=None, params=None, show=True, value_only=False):
+    """Run a single metric and cache the results
+
+    Args:
+        metric_id (str): The metric id (e.g. 'validmind.unit_metrics.classification.sklearn.F1')
+        inputs (dict): A dictionary of the metric inputs
+        params (dict): A dictionary of the metric parameters
+        show (bool): Whether to display the results
+        value_only (bool): Whether to return only the value
+    """
+    inputs = inputs or {}
+    params = params or {}
+
+    cache_key = get_metric_cache_key(metric_id, params, inputs)
+
+    if cache_key not in unit_metric_results_cache:
+        metric = load_metric(metric_id)
+        _inputs, _params = _inspect_signature(metric)
+
+        result = metric(
+            **{k: v for k, v in inputs.items() if k in _inputs.keys()},
+            **{k: v for k, v in params.items() if k in _params.keys()},
+        )
+        unit_metric_results_cache[cache_key] = (result, list(_inputs.keys()))
+
+    value = unit_metric_results_cache[cache_key][0]
+
+    if value_only:
+        return value
+
+    output_template = f"""
+    <table>
+        <thead>
+            <tr>
+                <th>Metric</th>
+                <th>Value</th>
+            </tr>
+        </thead>
+        <tbody>
+            <tr>
+                <td><strong>{test_id_to_name(metric_id)}</strong></td>
+                <td>{value:.4f}</td>
+            </tr>
+        </tbody>
+    </table>
+    <style>
+        th, td {{
+            padding: 5px;
+            text-align: left;
+        }}
+    </style>
+    """
+    result = _build_result(
+        results=value,
+        test_id=metric_id,
+        description="",
+        output_template=output_template,
+        inputs=unit_metric_results_cache[cache_key][1],
+    )
 
-    # Initialize the metric
-    metric = metric_class(test_id=metric_id, inputs=TestInput(inputs), params=params)
+    # in case the user tries to log the result object
+    def log(self):
+        raise Exception(
+            "Cannot log unit metrics directly..."
+            "You can run this unit metric as part of a composite metric and log that"
+        )
 
-    # Run the metric
-    result = metric.run()
+    result.log = log
 
-    unit_metric_results_cache[cache_key] = result
+    if show:
+        result.show()
 
     return result

validmind/unit_metrics/classification/sklearn/Accuracy.py

@@ -0,0 +1,14 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.metrics import accuracy_score
+
+from validmind import tags, tasks
+
+
+@tags("classification", "sklearn", "unit_metric")
+@tasks("classification")
+def Accuracy(dataset, model):
+    """Calculates the accuracy of a model"""
+    return accuracy_score(dataset.y, dataset.y_pred(model))