validmind 2.0.7__py3-none-any.whl → 2.1.0__py3-none-any.whl

validmind/tests/model_validation/statsmodels/LogisticRegCumulativeProb.py

@@ -1,172 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import numpy as np
-import pandas as pd
-import plotly.graph_objects as go
-from plotly.subplots import make_subplots
-
-from validmind.vm_models import Figure, Metric
-
-
-@dataclass
-class LogisticRegCumulativeProb(Metric):
-    """
-    Visualizes cumulative probabilities of positive and negative classes for both training and testing in logistic
-    regression models.
-
-    **Purpose**: This metric is utilized to evaluate the distribution of predicted probabilities for positive and
-    negative classes in a logistic regression model. It's not solely intended to measure the model's performance but
-    also provides a visual assessment of the model's behavior by plotting the cumulative probabilities for positive and
-    negative classes across both the training and test datasets.
-
-    **Test Mechanism**: The logistic regression model is evaluated by first computing the predicted probabilities for
-    each instance in both the training and test datasets, which are then added as a new column in these sets. The
-    cumulative probabilities for positive and negative classes are subsequently calculated and sorted in ascending
-    order. Cumulative distributions of these probabilities are created for both positive and negative classes across
-    both training and test datasets. These cumulative probabilities are represented visually in a plot, containing two
-    subplots - one for the training data and the other for the test data, with lines representing cumulative
-    distributions of positive and negative classes.
-
-    **Signs of High Risk**:
-    - Imbalanced distribution of probabilities for either positive or negative classes.
-    - Notable discrepancies or significant differences between the cumulative probability distributions for the
-    training data versus the test data.
-    - Marked discrepancies or large differences between the cumulative probability distributions for positive and
-    negative classes.
-
-    **Strengths**:
-    - It offers not only numerical probabilities but also provides a visual illustration of data, which enhances the
-    ease of understanding and interpreting the model's behavior.
-    - Allows for the comparison of model's behavior across training and testing datasets, providing insights about how
-    well the model is generalized.
-    - It differentiates between positive and negative classes and their respective distribution patterns, which can aid
-    in problem diagnosis.
-
-    **Limitations**:
-    - Exclusive to classification tasks and specifically to logistic regression models.
-    - Graphical results necessitate human interpretation and may not be directly applicable for automated risk
-    detection.
-    - The method does not give a solitary quantifiable measure of model risk, rather it offers a visual representation
-    and broad distributional information.
-    - If the training and test datasets are not representative of the overall data distribution, the metric could
-    provide misleading results.
-    """
-
-    name = "logistic_reg_cumulative_prob"
-    required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["logistic_regression", "visualization"],
-    }
-    default_params = {"title": "Cumulative Probabilities"}
-
-    @staticmethod
-    def compute_probabilities(model, X):
-        """
-        Predict probabilities and add them as a new column in X
-        """
-        probabilities = model.predict(X)
-        X["probabilities"] = probabilities
-        return X
-
-    @staticmethod
-    def plot_cumulative_prob(df_train, df_test, prob_col, target_col, title):
-        # Separate probabilities based on target column
-        train_0 = np.sort(df_train[df_train[target_col] == 0][prob_col])
-        train_1 = np.sort(df_train[df_train[target_col] == 1][prob_col])
-        test_0 = np.sort(df_test[df_test[target_col] == 0][prob_col])
-        test_1 = np.sort(df_test[df_test[target_col] == 1][prob_col])
-
-        # Calculate cumulative distributions
-        cumulative_train_0 = np.cumsum(train_0) / np.sum(train_0)
-        cumulative_train_1 = np.cumsum(train_1) / np.sum(train_1)
-        cumulative_test_0 = np.cumsum(test_0) / np.sum(test_0)
-        cumulative_test_1 = np.cumsum(test_1) / np.sum(test_1)
-
-        # Create subplot
-        fig = make_subplots(rows=1, cols=2, subplot_titles=("Train Data", "Test Data"))
-
-        # Create line plots for training data
-        trace_train_0 = go.Scatter(
-            x=train_0,
-            y=cumulative_train_0,
-            mode="lines",
-            name=f"Train {target_col} = 0",
-        )
-        trace_train_1 = go.Scatter(
-            x=train_1,
-            y=cumulative_train_1,
-            mode="lines",
-            name=f"Train {target_col} = 1",
-        )
-
-        # Create line plots for testing data
-        trace_test_0 = go.Scatter(
-            x=test_0, y=cumulative_test_0, mode="lines", name=f"Test {target_col} = 0"
-        )
-        trace_test_1 = go.Scatter(
-            x=test_1, y=cumulative_test_1, mode="lines", name=f"Test {target_col} = 1"
-        )
-
-        # Add traces to the subplots
-        fig.add_trace(trace_train_0, row=1, col=1)
-        fig.add_trace(trace_train_1, row=1, col=1)
-        fig.add_trace(trace_test_0, row=1, col=2)
-        fig.add_trace(trace_test_1, row=1, col=2)
-
-        # Update layout
-        fig.update_layout(title_text=title)
-
-        return fig
-
-    def run(self):
-        model = (
-            self.inputs.model[0]
-            if isinstance(self.inputs.model, list)
-            else self.inputs.model
-        )
-
-        target_column = self.datasets[0].target_column
-        title = self.params["title"]
-
-        # Create a copy of training and testing dataframes
-        df_train = self.datasets[0].df.copy()
-        df_test = self.datasets[1].df.copy()
-
-        # Drop target_column to create feature dataframes
-        X_train = df_train.drop(columns=[target_column])
-        X_test = df_test.drop(columns=[target_column])
-
-        # Subset only target_column to create target dataframes
-        y_train = df_train[[target_column]]
-        y_test = df_test[[target_column]]
-
-        X_train = self.compute_probabilities(model, X_train)
-        X_test = self.compute_probabilities(model, X_test)
-
-        df_train = pd.concat([X_train, y_train], axis=1)
-        df_test = pd.concat([X_test, y_test], axis=1)
-
-        fig = self.plot_cumulative_prob(
-            df_train, df_test, "probabilities", target_column, title
-        )
-
-        return self.cache_results(
-            metric_value={
-                "cum_prob": {
-                    "train_probs": list(X_train["probabilities"]),
-                    "test_probs": list(X_test["probabilities"]),
-                },
-            },
-            figures=[
-                Figure(
-                    for_object=self,
-                    key="cum_prob",
-                    figure=fig,
-                )
-            ],
-        )

validmind/tests/model_validation/statsmodels/ScorecardBucketHistogram.py

@@ -1,181 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import numpy as np
-import pandas as pd
-import plotly.graph_objects as go
-
-from validmind.vm_models import Figure, Metric
-
-
-@dataclass
-class ScorecardBucketHistogram(Metric):
-    """
-    Evaluates and visualizes distribution of risk categories in a classification model's scores, useful in credit risk
-    assessment.
-
-    **Purpose**: The 'Scorecard Bucket Histogram' is employed as a metric to evaluate the performance of a
-    classification model, specifically in credit risk assessment. It categorizes model scores into different rating
-    classes, and visualizes the distribution of scores or probabilities within each class. It essentially measures how
-    different risk categories (classes) are distributed in the model scores and provides insight into the model's
-    classification ability. This makes it particularly useful in credit scoring and risk modeling where understanding
-    the probability of default is critical.
-
-    **Test Mechanism**: The test works by computing the probabilities for each record in the test and train dataset
-    using the model's predict function. Subsequently, it calculates the scores using a formula incorporating target
-    score, target odds, and points to double odds (PDO). The scores are then bucketed into predefined rating classes
-    (such as 'A', 'B', 'C', 'D') and plotted in a histogram for both the train and test datasets. The target score,
-    target odds, points to double the odds (PDO), and rating classes are customizable parameters, providing flexibility
-    in test metrics based on differing model or industry norms.
-
-    **Signs of High Risk**:
-
-    - Disproportionate scores within rating classes
-    - Excessive overlap between classes
-    - Inconsistent distribution of scores between the training and testing datasets
-
-    If the model is accurately classifying and risk is being evenly distributed, we would anticipate smooth and
-    relatively balanced histograms within classes.
-
-    **Strengths**:
-
-    - Provides a quick visual snapshot of score distribution
-    - Breaks down complex predictions into simple, understandable classes, making it easily interpretable for both
-    technical and non-technical audiences
-    - Caters to customization of parameters
-    - Gives ownership of the class definitions to the user
-    - Useful in the field of credit risk, providing a clear understanding of which class or 'bucket' a potential
-    borrower belongs to
-
-    **Limitations**:
-
-    - Relies on manual setting of classes and other parameters (like target score, target odds, and PDO), potentially
-    leading to arbitrary classifications and potential bias if not judiciously performed
-    - Effectiveness can be limited with non-tabular data
-    - Doesn't provide a numerical value easily compared across different models or runs as the output is primarily
-    visual
-    - Might not present a complete view of model performance and should be used in conjunction with other metrics
-    """
-
-    name = "scorecard_bucket_histogram"
-    required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["tabular_data", "visualization", "credit_risk"],
-    }
-    default_params = {
-        "title": "Distribution of Scores by Rating Classes",
-        "target_score": 600,
-        "target_odds": 50,
-        "pdo": 20,
-        "rating_classes": ["A", "B", "C", "D"],
-    }
-
-    @staticmethod
-    def compute_probabilities(model, X):
-        """
-        Predict probabilities and add them as a new column in X
-        """
-        probabilities = model.predict(X)
-        X["probabilities"] = probabilities
-        return X
-
-    @staticmethod
-    def compute_scores(X, target_score, target_odds, pdo):
-        X_copy = X.copy()
-        factor = pdo / np.log(2)
-        offset = target_score - (factor * np.log(target_odds))
-
-        X_copy["score"] = offset + factor * np.log(
-            X_copy["probabilities"] / (1 - X_copy["probabilities"])
-        )
-
-        return X_copy
-
-    @staticmethod
-    def plot_score_bucket_histogram(df, score_col, title, rating_classes):
-        df["bucket"] = pd.cut(
-            df[score_col], bins=len(rating_classes), labels=rating_classes, right=False
-        )
-
-        fig = go.Figure()
-
-        color_scale = [[0.0, "rgba(178, 24, 43, 1)"], [1.0, "rgba(33, 102, 172, 1)"]]
-
-        for bucket in rating_classes:
-            df_bucket = df[df["bucket"] == bucket]
-            bucket_values = df_bucket[score_col]
-            fig.add_trace(
-                go.Histogram(
-                    x=bucket_values,
-                    name=bucket,
-                    opacity=0.6,
-                )
-            )
-
-        fig.update_layout(
-            title_text=title,
-            xaxis_title="",
-            yaxis_title="Frequency",
-            barmode="overlay",
-            coloraxis=dict(colorscale=color_scale, colorbar=dict(title="")),
-        )
-
-        return fig
-
-    def run(self):
-        title = self.params["title"]
-        target_score = self.params["target_score"]
-        target_odds = self.params["target_odds"]
-        pdo = self.params["pdo"]
-        rating_classes = self.params["rating_classes"]
-
-        X_train = self.inputs.datasets[0].x.copy()
-        X_test = self.inputs.datasets[1].x.copy()
-
-        X_train_probs = self.compute_probabilities(self.inputs.model, X_train)
-        X_test_probs = self.compute_probabilities(self.inputs.model, X_test)
-
-        df_train_scores = self.compute_scores(
-            X_train_probs, target_score, target_odds, pdo
-        )
-        df_test_scores = self.compute_scores(
-            X_test_probs, target_score, target_odds, pdo
-        )
-
-        fig_train = self.plot_score_bucket_histogram(
-            df_train_scores,
-            "score",
-            title + " - Train Data",
-            rating_classes,
-        )
-        fig_test = self.plot_score_bucket_histogram(
-            df_test_scores,
-            "score",
-            title + " - Test Data",
-            rating_classes,
-        )
-
-        return self.cache_results(
-            metric_value={
-                "score_distribution": {
-                    "train_scores": list(df_train_scores["score"]),
-                    "test_scores": list(df_test_scores["score"]),
-                },
-            },
-            figures=[
-                Figure(
-                    for_object=self,
-                    key="score_distribution_train",
-                    figure=fig_train,
-                ),
-                Figure(
-                    for_object=self,
-                    key="score_distribution_test",
-                    figure=fig_test,
-                ),
-            ],
-        )

validmind/tests/model_validation/statsmodels/ScorecardProbabilitiesHistogram.py

@@ -1,175 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import numpy as np
-import pandas as pd
-import plotly.graph_objects as go
-
-from validmind.vm_models import Figure, Metric
-
-
-@dataclass
-class ScorecardProbabilitiesHistogram(Metric):
-    """
-    Evaluates risk classification of a model by visualizing the distribution of default probability across score
-    buckets.
-
-    **Purpose**: The Scorecard Probabilities Histogram, a specific metric used within the credit risk domain, is
-    designed to evaluate and visualize risk classification of a model. It aims at examining the distribution of the
-    probability of default across varied score buckets, with the score buckets being categories that entities (e.g.,
-    loan applicants) are classed under based on their predicted default risks. The key idea is to ensure that the model
-    accurately classifies entities into appropriate risk categories (score buckets) and aptly represents their default
-    probabilities.
-
-    **Test Mechanism**: The mechanism behind the Scorecard Probabilities Histogram includes several steps. It starts
-    with the calculation of default probabilities by the 'compute_probabilities' method, where the resulting
-    probability is added as a fresh column to the input dataset. Following that, scores are computed using these
-    probabilities, a target score, target odds, and a Points to Double the odds (pdo) factor by the 'compute_scores'
-    method. These scores are then bucketed via the 'compute_buckets' method. A histogram is then plotted for each score
-    bucket, with default probabilities as the x-axis and their frequency as the y-axis - implemented within the
-    'plot_probabilities_histogram' method. This entire process is executed distinctly for both training and testing
-    datasets.
-
-    **Signs of High Risk**:
-    - A significant overlap of different score buckets in the histogram indicates that the model is not efficiently
-    distinguishing between various risk categories.
-    - If very high or low probabilities are commonplace across all buckets, the model's predictions could be skewed.
-
-    **Strengths**:
-    - The Scorecard Probabilities Histogram allows for the visualization and analysis of the predicted default risk
-    distribution across different risk classes, thereby facilitating a visual inspection of the model's performance and
-    calibration for various risk categories.
-    - It provides a means to visualize how these classifications are distributed on the training and testing datasets
-    separately, contributing to a better comprehension of model generalization.
-
-    **Limitations**:
-    - The Scorecard Probabilities Histogram assumes linear and equally spaced risk categories, which might not always
-    hold true.
-    - If there are too few or too many score buckets, the visualization may not convey sufficient information.
-    - While it effectively illustrates the distribution of probabilities, it does not provide adequate numerical
-    metrics or threshold to definitively evaluate the model's performance. A more accurate evaluation necessitates its
-    usage in conjunction with other metrics and tools including the confusion matrix, AUC-ROC, Precision, Recall, and
-    so forth.
-    """
-
-    name = "scorecard_probabilities_histogram"
-    required_inputs = ["model"]
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["tabular_data", "visualization", "credit_risk"],
-    }
-    default_params = {
-        "title": "Probability of Default by Score Bucket",
-        "target_score": 600,
-        "target_odds": 50,
-        "pdo": 20,
-        "score_buckets": ["A", "B", "C", "D"],
-    }
-
-    @staticmethod
-    def compute_probabilities(model, X):
-        """
-        Predict probabilities and add them as a new column in X
-        """
-        probabilities = model.predict(X)
-        X["probabilities"] = probabilities
-        return X
-
-    @staticmethod
-    def compute_scores(X, target_score, target_odds, pdo):
-        X_copy = X.copy()
-        factor = pdo / np.log(2)
-        offset = target_score - (factor * np.log(target_odds))
-
-        X_copy["score"] = offset + factor * np.log(
-            X_copy["probabilities"] / (1 - X_copy["probabilities"])
-        )
-
-        return X_copy
-
-    @staticmethod
-    def compute_buckets(X, score_buckets):
-        X["bucket"] = pd.qcut(X["score"], q=len(score_buckets), labels=score_buckets)
-        return X
-
-    @staticmethod
-    def plot_probabilities_histogram(df, title, score_buckets):
-        fig = go.Figure()
-
-        for bucket in score_buckets:
-            df_bucket = df[df["bucket"] == bucket]
-            bucket_values = df_bucket["probabilities"]
-            fig.add_trace(
-                go.Histogram(
-                    x=bucket_values,
-                    name=bucket,
-                    opacity=0.6,
-                )
-            )
-
-        fig.update_layout(
-            title_text=title,
-            xaxis_title="Probability",
-            yaxis_title="Frequency",
-            barmode="overlay",
-        )
-
-        return fig
-
-    def run(self):
-        title = self.params["title"]
-        target_score = self.params["target_score"]
-        target_odds = self.params["target_odds"]
-        pdo = self.params["pdo"]
-        score_buckets = self.params["score_buckets"]
-
-        X_train = self.inputs.datasets[0].x.copy()
-        X_test = self.inputs.datasets[1].x.copy()
-
-        X_train_probs = self.compute_probabilities(self.inputs.model, X_train)
-        X_test_probs = self.compute_probabilities(self.inputs.model, X_test)
-
-        df_train_scores = self.compute_scores(
-            X_train_probs, target_score, target_odds, pdo
-        )
-        df_test_scores = self.compute_scores(
-            X_test_probs, target_score, target_odds, pdo
-        )
-
-        df_train_buckets = self.compute_buckets(df_train_scores, score_buckets)
-        df_test_buckets = self.compute_buckets(df_test_scores, score_buckets)
-
-        fig_train = self.plot_probabilities_histogram(
-            df_train_buckets,
-            title + " - Train Data",
-            score_buckets,
-        )
-        fig_test = self.plot_probabilities_histogram(
-            df_test_buckets,
-            title + " - Test Data",
-            score_buckets,
-        )
-
-        return self.cache_results(
-            metric_value={
-                "probability_distribution": {
-                    "train_probs": list(df_train_buckets["probabilities"]),
-                    "test_probs": list(df_test_buckets["probabilities"]),
-                },
-            },
-            figures=[
-                Figure(
-                    for_object=self,
-                    key="probability_distribution_train",
-                    figure=fig_train,
-                ),
-                Figure(
-                    for_object=self,
-                    key="probability_distribution_test",
-                    figure=fig_test,
-                ),
-            ],
-        )

validmind/unit_metrics/sklearn/classification/Accuracy.py

@@ -1,22 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-from sklearn.metrics import accuracy_score
-
-from validmind.vm_models import UnitMetric
-
-
-@dataclass
-class Accuracy(UnitMetric):
-    required_inputs = ["dataset", "model"]
-
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
-
-        value = accuracy_score(y_true, y_pred, **self.params)
-
-        return self.cache_results(metric_value=value)

validmind/unit_metrics/sklearn/classification/F1.py

@@ -1,24 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-from sklearn.metrics import f1_score
-
-from validmind.vm_models import UnitMetric
-
-
-@dataclass
-class F1(UnitMetric):
-    required_inputs = ["dataset", "model"]
-
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
-
-        value = f1_score(y_true, y_pred, **self.params)
-
-        return self.cache_results(
-            metric_value=value,
-        )

validmind/unit_metrics/sklearn/classification/Precision.py

@@ -1,24 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-from sklearn.metrics import precision_score
-
-from validmind.vm_models import UnitMetric
-
-
-@dataclass
-class Precision(UnitMetric):
-    required_inputs = ["dataset", "model"]
-
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
-
-        value = precision_score(y_true, y_pred, **self.params)
-
-        return self.cache_results(
-            metric_value=value,
-        )

validmind/unit_metrics/sklearn/classification/ROC_AUC.py

@@ -1,22 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-from sklearn.metrics import roc_auc_score
-
-from validmind.vm_models import UnitMetric
-
-
-@dataclass
-class ROC_AUC(UnitMetric):
-    required_inputs = ["dataset", "model"]
-
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
-
-        value = roc_auc_score(y_true, y_pred, **self.params)
-
-        return self.cache_results(metric_value=value)

validmind/unit_metrics/sklearn/classification/Recall.py

@@ -1,22 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-from sklearn.metrics import recall_score
-
-from validmind.vm_models import UnitMetric
-
-
-@dataclass
-class Recall(UnitMetric):
-    required_inputs = ["dataset", "model"]
-
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
-
-        value = recall_score(y_true, y_pred, **self.params)
-
-        return self.cache_results(metric_value=value)