validmind 2.2.6__py3-none-any.whl → 2.3.3__py3-none-any.whl

validmind/tests/data_validation/PiTPDHistogram.py

@@ -1,152 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import pandas as pd
-import plotly.graph_objects as go
-from plotly.subplots import make_subplots
-
-from validmind.vm_models import Figure, Metric
-
-
-@dataclass
-class PiTPDHistogram(Metric):
-    """
-    Assesses credit risk prediction accuracy of a model by comparing actual and predicted defaults at a chosen point in
-    time.
-
-    **Purpose**: The PiTPDHistogram metric uses Probability of Default (PD) calculations for individual instances
-    within both training and test data sets in order to assess a model's proficiency in predicting credit risk. A
-    distinctive point in time (PiT) is chosen for these PD calculations, and the results for both actual and predicted
-    defaults are presented in histogram form. This visualization is aimed at simplifying the understanding of model
-    prediction accuracy.
-
-    **Test Mechanism**: Instances are categorized into two groups - those for actual defaults and those for predicted
-    defaults, with '1' indicating a default and '0' indicating non-default. PD is calculated for each instance, and
-    based on these calculations, two histograms are created, one for actual defaults and one for predicted defaults. If
-    the predicted default frequency matches that of the actual defaults, the model's performance is deemed effective.
-
-    **Signs of High Risk**:
-    - Discrepancies between the actual and predicted default histograms may suggest model inefficiency.
-    - Variations in histogram shapes or divergences in default probability distributions could be concerning.
-    - Significant mismatches in peak default probabilities could also be red flags.
-
-    **Strengths**:
-    - Provides a visual comparison between actual and predicted defaults, aiding in the understanding of model
-    performance.
-    - Helps reveal model bias and areas where the model's performance could be improved.
-    - Easier to understand than purely numerical evaluations or other complicated visualization measures.
-
-    **Limitations**:
-    - The metric remains largely interpretive and subjective, as the extent and relevance of visual discrepancies often
-    need to be evaluated manually, leading to potentially inconsistent results across different analyses.
-    - This metric alone may not capture all the complexities and nuances of model performance.
-    - The information provided is limited to a specific point in time, potentially neglecting the model's performance
-    under various circumstances or different time periods.
-    """
-
-    name = "pit_pd_histogram"
-    required_context = ["dataset"]
-    default_params = {"title": "Histogram of PiT Probability of Default"}
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["tabular_data", "visualization", "credit_risk"],
-    }
-
-    @staticmethod
-    def plot_pit_pd_histogram(
-        df,
-        default_column,
-        predicted_default_column,
-        default_probabilities_column,
-        title,
-        point_in_time_date,
-    ):
-        fig = make_subplots(
-            rows=1, cols=2, subplot_titles=("Observed Default", "Predicted Default")
-        )
-
-        observed_data_0 = df[df[default_column] == 0][default_probabilities_column]
-        observed_data_1 = df[df[default_column] == 1][default_probabilities_column]
-
-        predicted_data_0 = df[df[predicted_default_column] == 0][
-            default_probabilities_column
-        ]
-        predicted_data_1 = df[df[predicted_default_column] == 1][
-            default_probabilities_column
-        ]
-
-        fig.add_trace(
-            go.Histogram(x=observed_data_0, opacity=0.75, name="Observed Default = 0"),
-            row=1,
-            col=1,
-        )
-        fig.add_trace(
-            go.Histogram(x=observed_data_1, opacity=0.75, name="Observed Default = 1"),
-            row=1,
-            col=1,
-        )
-
-        fig.add_trace(
-            go.Histogram(
-                x=predicted_data_0, opacity=0.75, name="Predicted Default = 0"
-            ),
-            row=1,
-            col=2,
-        )
-        fig.add_trace(
-            go.Histogram(
-                x=predicted_data_1, opacity=0.75, name="Predicted Default = 1"
-            ),
-            row=1,
-            col=2,
-        )
-
-        title += f" (PiT: {point_in_time_date.strftime('%d %b %Y')})"
-        fig.update_layout(barmode="overlay", title_text=title)
-
-        return fig
-
-    def run(self):
-        df = self.inputs.dataset.df
-        default_column = self.params["default_column"]
-        predicted_default_column = self.params["predicted_default_column"]
-        default_probabilities_column = self.params["default_probabilities_column"]
-        point_in_time_column = self.params["point_in_time_column"]
-
-        title = self.params["title"]
-
-        point_in_time_date = pd.to_datetime(df[point_in_time_column].iloc[0])
-
-        fig = self.plot_pit_pd_histogram(
-            df,
-            default_column,
-            predicted_default_column,
-            default_probabilities_column,
-            title,
-            point_in_time_date,
-        )
-
-        return self.cache_results(
-            metric_value={
-                "prob_histogram": {
-                    "observed_probs": list(
-                        df[df[default_column] == 1][default_probabilities_column]
-                    ),
-                    "predicted_probs": list(
-                        df[df[predicted_default_column] == 1][
-                            default_probabilities_column
-                        ]
-                    ),
-                },
-            },
-            figures=[
-                Figure(
-                    for_object=self,
-                    key="prob_histogram",
-                    figure=fig,
-                )
-            ],
-        )

validmind/tests/model_validation/statsmodels/ADFTest.py

@@ -1,88 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-from statsmodels.tsa.stattools import adfuller
-
-from validmind.vm_models import ThresholdTest, ThresholdTestResult
-
-
-@dataclass
-class ADFTest(ThresholdTest):
-    """
-    Assesses the stationarity of time series data using the Augmented Dickey-Fuller (ADF) test.
-
-    **Purpose**: The Augmented Dickey-Fuller (ADF) metric test is designed to evaluate the presence of a unit root in a
-    time series. This essentially translates to assessing the stationarity of a time series dataset. This is vital in
-    time series analysis, regression tasks, and forecasting, as these often need the data to be stationary.
-
-    **Test Mechanism**: This test application utilizes the "adfuller" function from Python's “statsmodels” library. It
-    applies this function to each column of the training dataset, subsequently calculating the ADF statistic, p-value,
-    the number of lags used, and the number of observations in the sample for each column. If a column's p-value is
-    lower than the predetermined threshold (usually 0.05), the series is considered stationary, and the test is deemed
-    passed for that column.
-
-    **Signs of High Risk**:
-    - A p-value that surpasses the threshold value indicates a high risk or potential model performance issue.
-    - A high p-value suggests that the null hypothesis (of a unit root being present) cannot be rejected. This in turn
-    suggests that the series is non-stationary which could potentially yield unreliable and falsified results for the
-    model's performance and forecast.
-
-    **Strengths**:
-    - Archetypal Test for Stationarity: The ADF test is a comprehensive approach towards testing the stationarity of
-    time series data. Such testing is vital for many machine learning and statistical models.
-    - Detailed Output: The function generates detailed output, including the number of lags used and the number of
-    observations, which adds to understanding a series’ behaviour.
-
-    **Limitations**:
-    - Dependence on Threshold: The result of this test freights heavily on the threshold chosen. Hence, an imprudent
-    threshold value might lead to false acceptance or rejection of the null hypothesis.
-    - Not Effective for Trending Data: The test suffers when it operates under the assumption that the data does not
-    encapsulate any deterministic trend. In the presence of such a trend, it might falsely identify a series as
-    non-stationary.
-    - Potential for False Positives: The ADF test especially in the case of larger datasets, tends to reject the null
-    hypothesis, escalating the chances of false positives.
-    """
-
-    name = "adf_test"
-    required_inputs = ["dataset"]
-    default_params = {"threshold": 0.05}
-    metadata = {
-        "task_types": ["regression"],
-        "tags": [
-            "time_series_data",
-            "statsmodels",
-            "forecasting",
-            "statistical_test",
-            "stationarity",
-        ],
-    }
-
-    def run(self):
-        x_train = self.inputs.dataset.df
-
-        results = []
-        for col in x_train.columns:
-            # adf_values[col] = adfuller(x_train[col].values)
-            adf, pvalue, usedlag, nobs, critical_values, icbest = adfuller(
-                x_train[col].values
-            )
-
-            col_passed = pvalue < self.params["threshold"]
-            results.append(
-                ThresholdTestResult(
-                    column=col,
-                    passed=col_passed,
-                    values={
-                        "adf": adf,
-                        "pvalue": pvalue,
-                        "usedlag": usedlag,
-                        "nobs": nobs,
-                        "icbest": icbest,
-                    },
-                )
-            )
-
-        return self.cache_results(results, passed=all([r.passed for r in results]))

validmind/tests/model_validation/statsmodels/FeatureImportanceAndSignificance.py

@@ -1,198 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import pandas as pd
-import plotly.graph_objects as go
-from sklearn.inspection import permutation_importance
-
-from validmind.logging import get_logger
-from validmind.vm_models import Figure, Metric
-
-logger = get_logger(__name__)
-
-
-@dataclass
-class FeatureImportanceAndSignificance(Metric):
-    """
-    Evaluates and visualizes the statistical significance and feature importance using regression and decision tree
-    models.
-
-    **Purpose**: The 'FeatureImportanceAndSignificance' test evaluates the statistical significance and the importance
-    of features in the context of the machine learning model. By comparing the p-values from a regression model and the
-    feature importances from a decision tree model, this test aids in determining the most significant variables from a
-    statistical and a machine learning perspective, assisting in feature selection during the model development process.
-
-    **Test Mechanism**: The test first compares the p-values from a regression model and the feature importances from a
-    decision tree model. These values are normalized to ensure a uniform comparison. The 'p_threshold' parameter is
-    used to determine what p-value is considered statistically significant and if the 'significant_only' parameter is
-    true, only features with p-values below this threshold are included in the final output. The output from this test
-    includes an interactive visualization displaying normalized p-values and the associated feature importances. The
-    test throws an error if it does not receive both a regression model and a decision tree model.
-
-    **Signs of High Risk**:
-    - Exceptionally high or low p-values, which suggest that a feature may not be significant or meaningful in the
-    context of the model.
-    - If many variables with small feature importance values have significant p-values, this could indicate that the
-    model might be overfitting.
-
-    **Strengths**:
-    - Combines two perspectives statistical significance (p-values) and feature importance (decision tree model),
-    making it a robust feature selection test.
-    - Provides an interactive visualization making it easy to interpret and understand the results.
-
-    **Limitations**:
-    - The test only works with a regression model and a decision tree model which may limit its applicability.
-    - The test does not take into account potential correlations or causative relationships between features which may
-    lead to misinterpretations of significance and importance.
-    - Over-reliance on the p-value as a cut-off for feature significance can be seen as arbitrary and may not truly
-    reflect the real-world importance of the feature.
-    """
-
-    name = "feature_importance_and_significance"
-    required_inputs = ["models"]
-    default_params = {
-        "fontsize": 10,
-        "p_threshold": 0.05,
-        "significant_only": False,
-        "figure_height": 800,
-        "bar_width": 0.3,
-    }
-    metadata = {
-        "task_types": ["regression"],
-        "tags": [
-            "statsmodels",
-            "feature_importance",
-            "statistical_test",
-            "visualization",
-        ],
-    }
-
-    def compute_p_values_and_feature_importances(
-        self, regression_model, decision_tree_model
-    ):
-        p_values = regression_model.model.pvalues
-        feature_importances = permutation_importance(
-            decision_tree_model.model,
-            decision_tree_model.train_ds.x,
-            decision_tree_model.train_ds.y,
-            random_state=0,
-            n_jobs=-2,
-        ).importances_mean
-
-        p_values = p_values / max(p_values)
-        feature_importances = feature_importances / max(feature_importances)
-
-        return p_values, feature_importances
-
-    def create_dataframe(
-        self,
-        p_values,
-        feature_importances,
-        regression_model,
-        significant_only,
-        p_threshold,
-    ):
-        df = pd.DataFrame(
-            {
-                "Normalized p-value": p_values,
-                "Normalized Feature Importance": feature_importances,
-            },
-            index=regression_model.train_ds.x_df().columns,
-        )
-
-        if significant_only:
-            df = df[df["Normalized p-value"] <= p_threshold]
-
-        df = df.sort_values(by="Normalized Feature Importance", ascending=True)
-
-        return df
-
-    def create_figure(self, df, fontsize, figure_height, bar_width):
-        fig = go.Figure()
-
-        title_text = (
-            "Significant Features (p-value <= {0})".format(self.params["p_threshold"])
-            if self.params["significant_only"]
-            else "All Features"
-        )
-
-        fig.update_layout(
-            title=title_text,
-            barmode="group",
-            height=figure_height,
-            yaxis=dict(tickfont=dict(size=fontsize)),
-            xaxis=dict(title="Normalized Value", titlefont=dict(size=fontsize)),
-        )
-
-        fig.add_trace(
-            go.Bar(
-                y=df.index,
-                x=df["Normalized p-value"],
-                name="Normalized p-value",
-                orientation="h",
-                marker=dict(color="skyblue"),
-                width=bar_width,
-            )
-        )
-
-        fig.add_trace(
-            go.Bar(
-                y=df.index,
-                x=df["Normalized Feature Importance"],
-                name="Normalized Feature Importance",
-                orientation="h",
-                marker=dict(color="orange"),
-                width=bar_width,
-            )
-        )
-
-        return fig
-
-    def run(self):
-        fontsize = self.params["fontsize"]
-        significant_only = self.params["significant_only"]
-        p_threshold = self.params["p_threshold"]
-        figure_height = self.params["figure_height"]
-        bar_width = self.params["bar_width"]
-
-        all_models = []
-
-        if self.inputs.models is not None:
-            all_models.extend(self.inputs.models)
-
-        if len(self.inputs.models) != 2:
-            raise ValueError("Two models must be provided")
-
-        regression_model = self.inputs.models[0]
-        decision_tree_model = self.inputs.models[1]
-
-        p_values, feature_importances = self.compute_p_values_and_feature_importances(
-            regression_model, decision_tree_model
-        )
-
-        df = self.create_dataframe(
-            p_values,
-            feature_importances,
-            regression_model,
-            significant_only,
-            p_threshold,
-        )
-
-        fig = self.create_figure(df, fontsize, figure_height, bar_width)
-
-        return self.cache_results(
-            figures=[
-                Figure(
-                    for_object=self,
-                    key=self.key,
-                    figure=fig,
-                    metadata={
-                        "model_regression": str(regression_model.model),
-                        "model_decision_tree": str(decision_tree_model.model),
-                    },
-                )
-            ]
-        )

validmind/tests/model_validation/statsmodels/PDRatingClassPlot.py

@@ -1,151 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import pandas as pd
-import plotly.graph_objects as go
-
-from validmind.vm_models import Figure, Metric
-
-
-@dataclass
-class PDRatingClassPlot(Metric):
-    """
-    Assesses and visualizes credit risk distribution across different rating classes within a dataset via default
-    probabilities.
-
-    **Purpose**: The purpose of the Probability of Default (PD) Rating Class Plot test is to measure and evaluate the
-    distribution of calculated default probabilities across different rating classes. This is critical for
-    understanding and inferring credit risk and can provide insights into how effectively the model is differentiating
-    between different risk levels in a credit dataset.
-
-    **Test Mechanism**: This metric is implemented via a visualization mechanism. It sorts the predicted probabilities
-    of defaults into user-defined rating classes defined in "rating_classes" in default parameters. When it has
-    classified the probabilities, it then calculates the average default rates within each rating class. Subsequently,
-    it produces bar plots for each of these rating classes, illustrating the average likelihood of a default within
-    each class. This process is executed separately for both the training and testing data sets. The classification of
-    predicted probabilities utilizes the pandas "cut" function, sorting and sectioning the data values into bins.
-
-    **Signs of High Risk**:
-
-    - If lower rating classes present higher average likelihoods of default than higher rating classes
-    - If there is poor differentiation between the averages across the different rating classes
-    - If the model generates a significant contrast between the likelihoods for the training set and the testing set,
-    suggestive of model overfitting
-
-    **Strengths**:
-
-    - Presents a clear visual representation of how efficient the model is at predicting credit risk across different
-    risk levels
-    - Allows for rapid identification and understanding of model performance per rating class
-    - Highlights potential overfitting issues by including both training and testing datasets in the analysis
-
-    **Limitations**:
-
-    - Making an incorrect choice for the number of rating classes, either oversimplifying or overcomplicating the
-    distribution of default rates
-    - Relying on the assumption that the rating classes are effective at differentiating risk levels and that the
-    boundaries between classes truly represent the risk distribution
-    - Not accounting for data set class imbalance, which could cause skewed average probabilities
-    - Inability to gauge the overall performance of the model only based on this metric, emphasizing the requirement of
-    combining it with other evaluation metrics
-    """
-
-    name = "pd_rating_class_plot"
-    required_inputs = ["model", "datasets"]
-
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["visualization", "credit_risk"],
-    }
-
-    default_params = {
-        "title": "PD by Rating Class",
-        "rating_classes": ["A", "B", "C", "D"],
-    }
-
-    @staticmethod
-    def plot_bucket_analysis(df, prob_col, target_col, title, rating_classes):
-        df["bucket"] = pd.cut(
-            df[prob_col], bins=len(rating_classes), labels=rating_classes, right=False
-        )
-        default_rate = df.groupby("bucket")[target_col].mean()
-
-        # Sort the data based on the order of rating_classes
-        sorted_data = sorted(
-            zip(rating_classes, default_rate),
-            key=lambda x: rating_classes.index(x[0]),
-        )
-        rating_classes_sorted, default_rate_sorted = zip(*sorted_data)
-
-        fig = go.Figure()
-
-        # Iterate through the sorted data and create a bar for each score bucket
-        for i, (bucket, rate) in enumerate(
-            zip(rating_classes_sorted, default_rate_sorted)
-        ):
-            fig.add_trace(go.Bar(x=[bucket], y=[rate], name=bucket))
-
-        fig.update_layout(
-            title_text=title,
-            xaxis_title="Rating Class",
-            yaxis_title="Probability of Default",
-            barmode="group",
-        )
-
-        return fig
-
-    def run(self):
-        target_column = self.inputs.model.train_ds.target_column
-        title = self.params["title"]
-        rating_classes = self.params["rating_classes"]
-
-        X_train = self.inputs.datasets[0].x.copy()
-        y_train = self.inputs.datasets[0].y.copy()
-        X_test = self.inputs.datasets[1].x.copy()
-        y_test = self.inputs.datasets[1].y.copy()
-
-        # Compute probabilities
-        X_train["probability"] = self.inputs.model.predict_proba(X_train)
-        X_test["probability"] = self.inputs.model.predict_proba(X_test)
-
-        df_train = pd.concat([X_train, y_train], axis=1)
-        df_test = pd.concat([X_test, y_test], axis=1)
-
-        fig_train = self.plot_bucket_analysis(
-            df_train,
-            "probability",
-            target_column,
-            title + " - Train Data",
-            rating_classes,
-        )
-        fig_test = self.plot_bucket_analysis(
-            df_test,
-            "probability",
-            target_column,
-            title + " - Test Data",
-            rating_classes,
-        )
-
-        return self.cache_results(
-            metric_value={
-                "bucket_analysis": {
-                    "train_probs": list(X_train["probability"]),
-                    "test_probs": list(X_test["probability"]),
-                },
-            },
-            figures=[
-                Figure(
-                    for_object=self,
-                    key="bucket_analysis_train",
-                    figure=fig_train,
-                ),
-                Figure(
-                    for_object=self,
-                    key="bucket_analysis_test",
-                    figure=fig_test,
-                ),
-            ],
-        )