validmind 2.5.8__py3-none-any.whl → 2.5.18__py3-none-any.whl

validmind/tests/model_validation/statsmodels/RegressionCoeffsPlot.py

@@ -1,135 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import pandas as pd
-import plotly.graph_objects as go
-from scipy import stats
-
-from validmind.errors import SkipTestError
-from validmind.vm_models import Figure, Metric
-
-
-@dataclass
-class RegressionCoeffsPlot(Metric):
-    """
-    Visualizes regression coefficients with 95% confidence intervals to assess predictor variables' impact on response
-    variable.
-
-    **Purpose**: The Regression Coefficients with Confidence Intervals plot and metric aims to understand the impact of
-    predictor variables on the response variable in question. This understanding is achieved via the visualization and
-    analysis of the regression model by presenting the coefficients derived from the model along with their associated
-    95% confidence intervals. By doing so, it offers insights into the variability and uncertainty associated with the
-    model's estimates.
-
-    **Test Mechanism**: The test begins by extracting the estimated coefficients and their related standard errors from
-    the regression model under test. It then calculates and draws confidence intervals based on a 95% confidence level
-    (a standard convention in statistics). These intervals provide a range wherein the true value can be expected to
-    fall 95% of the time if the same regression were re-run multiple times with samples drawn from the same population.
-    This information is then visualized as a bar plot, with the predictor variables and their coefficients on the
-    x-axis and y-axis respectively and the confidence intervals represented as error bars.
-
-    **Signs of High Risk**:
-    * If the calculated confidence interval contains the zero value, it could mean the feature/coefficient in question
-    doesn't significantly contribute to prediction in the model.
-    * If there are multiple coefficients exhibiting this behavior, it might raise concerns about overall model
-    reliability.
-    * Very wide confidence intervals might indicate high uncertainty in the associated coefficient estimates.
-
-    **Strengths**:
-    * This metric offers a simple and easily comprehendible visualization of the significance and impact of individual
-    predictor variables in a regression model.
-    * By including confidence intervals, it enables an observer to evaluate the uncertainty around each coefficient
-    estimate.
-
-    **Limitations**:
-    * The test is dependent on a few assumptions about the data, namely normality of residuals and independence of
-    observations, which may not always be true for all types of datasets.
-    * The test does not consider multi-collinearity (correlation among predictor variables), which can potentially
-    distort the model and make interpretation of coefficients challenging.
-    * The test's application is limited to regression tasks and tabular datasets and is not suitable for other types of
-    machine learning assignments or data structures.
-    """
-
-    name = "regression_coeffs_plot"
-    required_inputs = ["models"]
-    tasks = ["regression"]
-    tags = ["tabular_data", "visualization", "model_interpretation"]
-
-    @staticmethod
-    def plot_coefficients_with_ci(model, model_name):
-        # Extract estimated coefficients and standard errors
-        coefficients = model.regression_coefficients()
-        coef = pd.to_numeric(coefficients["coef"])
-        std_err = pd.to_numeric(coefficients["std err"])
-
-        # Calculate confidence intervals
-        confidence_level = 0.95  # 95% confidence interval
-        z_value = stats.norm.ppf((1 + confidence_level) / 2)  # Calculate Z-value
-        lower_ci = coef - z_value * std_err
-        upper_ci = coef + z_value * std_err
-
-        # Create a bar plot with confidence intervals
-        fig = go.Figure()
-
-        fig.add_trace(
-            go.Bar(
-                x=list(coefficients["Feature"].values),
-                y=coef,
-                name="Estimated Coefficients",
-                error_y=dict(
-                    type="data",
-                    symmetric=False,
-                    arrayminus=lower_ci,
-                    array=upper_ci,
-                    visible=True,
-                ),
-            )
-        )
-
-        fig.update_layout(
-            title=f"{model_name} Coefficients with Confidence Intervals",
-            xaxis_title="Predictor Variables",
-            yaxis_title="Coefficients",
-        )
-
-        return fig, {
-            "values": list(coef),
-            "lower_ci": list(lower_ci),
-            "upper_ci": list(upper_ci),
-        }
-
-    def run(self):
-        # Check models list is not empty
-        if not self.inputs.models:
-            raise ValueError("List of models must be provided in the models parameter")
-
-        all_models = []
-        all_figures = []
-        all_metric_values = []
-
-        if self.inputs.models is not None:
-            all_models.extend(self.inputs.models)
-
-        for i, model in enumerate(all_models):
-            if model.library != "statsmodels":
-                raise SkipTestError("Only statsmodels are supported for this metric")
-
-            model_name = f"Model {i+1}"
-
-            fig, metric_values = self.plot_coefficients_with_ci(model, model_name)
-            all_figures.append(
-                Figure(
-                    for_object=self,
-                    key=f"{model_name}_coefficients_ci_plot",
-                    figure=fig,
-                )
-            )
-            all_metric_values.append({"name": model_name, "metrics": metric_values})
-
-        return self.cache_results(
-            metric_value=all_metric_values,
-            figures=all_figures,
-        )

validmind/tests/model_validation/statsmodels/RegressionModelsCoeffs.py

@@ -1,103 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import pandas as pd
-
-from validmind.errors import SkipTestError
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
-
-
-@dataclass
-class RegressionModelsCoeffs(Metric):
-    """
-    Compares feature importance by evaluating and contrasting coefficients of different regression models.
-
-    **Purpose**: The 'RegressionModelsCoeffs' metric is utilized to evaluate and compare coefficients of different
-    regression models trained on the same dataset. By examining how each model weighted the importance of different
-    features during training, this metric provides key insights into which factors have the most impact on the model's
-    predictions and how these patterns differ across models.
-
-    **Test Mechanism**: The test operates by extracting the coefficients of each regression model using the
-    'regression_coefficients()' method. These coefficients are then consolidated into a dataframe, with each row
-    representing a model and columns corresponding to each feature's coefficient. It must be noted that this test is
-    exclusive to 'statsmodels' and 'R' models, other models will result in a 'SkipTestError'.
-
-    **Signs of High Risk**:
-    - Discrepancies in how different models weight the same features
-    - Unexpectedly high or low coefficients
-    - The test is inapplicable to certain models because they are not from 'statsmodels' or 'R' libraries
-
-    **Strengths**:
-    - Enables insight into the training process of different models
-    - Allows comparison of feature importance across models
-    - Through the review of feature coefficients, the test provides a more transparent evaluation of the model and
-    highlights significant weights and biases in the training procedure
-
-    **Limitations**:
-    - The test is only compatible with 'statsmodels' and 'R' regression models
-    - While the test provides contrast in feature weightings among models, it does not establish the most appropriate
-    or accurate weighting, thus remaining subject to interpretation
-    - It does not account for potential overfitting or underfitting of models
-    - The computed coefficients might not lead to effective performance on unseen data
-    """
-
-    name = "regression_models_coefficients"
-    required_inputs = ["models"]
-    tasks = ["regression"]
-    tags = ["model_comparison"]
-
-    def _build_model_summaries(self, all_coefficients):
-        all_models_df = pd.DataFrame()
-
-        for i, coefficients in enumerate(all_coefficients):
-            model_name = f"Model {i+1}"
-            # The coefficients summary object needs an additional "Model" column at the beginning
-            coefficients["Model"] = model_name
-            all_models_df = pd.concat([all_models_df, coefficients])
-
-        # Reorder columns to have 'Model' as the first column and reset the index
-        all_models_df = all_models_df.reset_index(drop=True)[
-            ["Model"] + [col for col in all_models_df.columns if col != "Model"]
-        ]
-
-        return all_models_df
-
-    def run(self):
-        # Check models list is not empty
-        if not self.inputs.models or len(self.inputs.models) == 0:
-            raise ValueError("List of models must be provided in the models parameter")
-
-        for model in self.inputs.models:
-            if model.library != "statsmodels":
-                raise SkipTestError(
-                    "Only statsmodels models are supported for this metric"
-                )
-
-        coefficients = [m.regression_coefficients() for m in self.inputs.models]
-        all_models_summary = self._build_model_summaries(coefficients)
-
-        return self.cache_results(
-            {
-                "coefficients_summary": all_models_summary.to_dict(orient="records"),
-            }
-        )
-
-    def summary(self, metric_value):
-        """
-        Build one table for summarizing the regression models' coefficients
-        """
-        coefficients_summary = metric_value["coefficients_summary"]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=coefficients_summary,
-                    metadata=ResultTableMetadata(
-                        title="Regression Models' Coefficients"
-                    ),
-                ),
-            ]
-        )

validmind/tests/model_validation/statsmodels/RunsTest.py

@@ -1,71 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from statsmodels.sandbox.stats.runs import runstest_1samp
-
-from validmind.vm_models import Metric
-
-
-class RunsTest(Metric):
-    """
-    Executes Runs Test on ML model to detect non-random patterns in output data sequence.
-
-    **Purpose**: The Runs Test is a statistical procedure used to determine whether the sequence of data extracted from
-    the ML model behaves randomly or not. Specifically, it analyzes runs, sequences of consecutive positives or
-    negatives, in the data to check if there are more or fewer runs than expected under the assumption of randomness.
-    This can be an indication of some pattern, trend, or cycle in the model's output which may need attention.
-
-    **Test Mechanism**: The testing mechanism applies the Runs Test from the statsmodels module on each column of the
-    training dataset. For every feature in the dataset, a Runs Test is executed, whose output includes a Runs Statistic
-    and P-value. A low P-value suggests that data arrangement in the feature is not likely to be random. The results
-    are stored in a dictionary where the keys are the feature names, and the values are another dictionary storing the
-    test statistic and the P-value for each feature.
-
-    **Signs of High Risk**:
-    - High risk is indicated when the P-value is close to zero.
-    - If the p-value is less than a predefined significance level (like 0.05), it suggests that the runs (series of
-    positive or negative values) in the model's output are not random and are longer or shorter than what is expected
-    under a random scenario.
-    - This would mean there's a high risk of non-random distribution of errors or model outcomes, suggesting potential
-    issues with the model.
-
-    **Strengths**:
-    - The strength of the Runs Test is that it's straightforward and fast for detecting non-random patterns in data
-    sequence.
-    - It can validate assumptions of randomness, which is particularly valuable for checking error distributions in
-    regression models, trendless time series data, and making sure a classifier doesn't favour one class over another.
-    - Moreover, it can be applied to both classification and regression tasks, making it versatile.
-
-    **Limitations**:
-    - The test assumes that the data is independently and identically distributed (i.i.d.), which might not be the case
-    for many real-world datasets.
-    - The conclusion drawn from the low p-value indicating non-randomness does not provide information about the type
-    or the source of the detected pattern.
-    - Also, it is sensitive to extreme values (outliers), and overly large or small run sequences can influence the
-    results.
-    - Furthermore, this test does not provide model performance evaluation; it is used to detect patterns in the
-    sequence of outputs only.
-    """
-
-    name = "runs_test"
-    required_inputs = ["dataset"]
-    tasks = ["classification", "regression"]
-    tags = ["tabular_data", "statistical_test", "statsmodels"]
-
-    def run(self):
-        """
-        Calculates the run test for each of the dataset features
-        """
-        x_train = self.inputs.dataset.df[self.inputs.dataset.feature_columns_numeric]
-
-        runs_test_values = {}
-        for col in x_train.columns:
-            runs_stat, runs_p_value = runstest_1samp(x_train[col].values)
-
-            runs_test_values[col] = {
-                "stat": runs_stat,
-                "pvalue": runs_p_value,
-            }
-
-        return self.cache_results(runs_test_values)