validmind 2.3.3__py3-none-any.whl → 2.4.0__py3-none-any.whl

validmind/tests/model_validation/sklearn/RegressionModelsPerformanceComparison.py

@@ -61,13 +61,11 @@ class RegressionModelsPerformanceComparison(Metric):
     name = "models_performance_comparison"
     required_inputs = ["dataset", "models"]
 
-    metadata = {
-        "task_types": ["regression"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["regression"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def regression_errors(self, y_true_test, y_pred_test):
         mae_test = mean_absolute_error(y_true_test, y_pred_test)

validmind/tests/model_validation/sklearn/RegressionR2Square.py

@@ -43,13 +43,11 @@ class RegressionR2Square(Metric):
 
     name = "regression_errors_r2_square"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["regression"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def summary(self, raw_results):
         """

validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.py

@@ -0,0 +1,63 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+from sklearn import metrics
+
+from validmind import tags, tasks
+from validmind.tests.model_validation.statsmodels.statsutils import adj_r2_score
+
+
+@tags("model_performance", "sklearn")
+@tasks("regression", "time_series_forecasting")
+def RegressionR2SquareComparison(datasets, models):
+    """
+    Compare R-Squared and Adjusted R-Squared values for each model and generate a summary table
+    with the results.
+
+    **Purpose**: The purpose of this function is to compare the R-Squared and Adjusted R-Squared values for different models applied to various datasets.
+
+    **Test Mechanism**: The function iterates through each dataset-model pair, calculates the R-Squared and Adjusted R-Squared values, and generates a summary table with these results.
+
+    **Signs of High Risk**:
+    - If the R-Squared values are significantly low, it could indicate that the model is not explaining much of the variability in the dataset.
+    - A significant difference between R-Squared and Adjusted R-Squared values might indicate that the model includes irrelevant features.
+
+    **Strengths**:
+    - Provides a quantitative measure of model performance in terms of variance explained.
+    - Adjusted R-Squared accounts for the number of predictors, making it a more reliable measure when comparing models with different numbers of features.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with `y`, `y_pred`, and `feature_columns` attributes.
+    - The function relies on `adj_r2_score` from the `statsmodels.statsutils` module, which should be correctly implemented and imported.
+    - Requires that `dataset.y_pred(model)` returns the predicted values for the model.
+
+    """
+    results_list = []
+
+    for dataset, model in zip(datasets, models):
+        dataset_name = dataset.input_id
+        model_name = model.input_id
+
+        y_true = dataset.y
+        y_pred = dataset.y_pred(model)  # Assuming dataset has X for features
+        y_true = y_true.astype(y_pred.dtype)
+
+        r2s = metrics.r2_score(y_true, y_pred)
+        X_columns = dataset.feature_columns
+        adj_r2 = adj_r2_score(y_true, y_pred, len(y_true), len(X_columns))
+
+        # Append results to the list
+        results_list.append(
+            {
+                "Model": model_name,
+                "Dataset": dataset_name,
+                "R-Squared": r2s,
+                "Adjusted R-Squared": adj_r2,
+            }
+        )
+
+    # Convert results list to a DataFrame
+    results_df = pd.DataFrame(results_list)
+    return results_df

validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py

@@ -75,20 +75,16 @@ class RobustnessDiagnosis(ThresholdTest):
         "scaling_factor_std_dev_list": [0.0, 0.1, 0.2, 0.3, 0.4, 0.5],
         "accuracy_decay_threshold": 4,
     }
-    metadata = {
-        "task_types": ["classification", "text_classification"],
-        "tags": [
-            "sklearn",
-            "binary_classification",
-            "multiclass_classification",
-            "model_diagnosis",
-            "visualization",
-        ],
-    }
-
-    default_metrics = {
-        "accuracy": metrics.accuracy_score,
-    }
+    tasks = ["classification", "text_classification"]
+    tags = [
+        "sklearn",
+        "binary_classification",
+        "multiclass_classification",
+        "model_diagnosis",
+        "visualization",
+    ]
+
+    default_metrics = {"accuracy": metrics.accuracy_score}
 
     def run(self):
         # Validate X std deviation parameter

validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py

@@ -65,16 +65,14 @@ class SHAPGlobalImportance(Metric):
 
     name = "shap"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["classification", "text_classification"],
-        "tags": [
-            "sklearn",
-            "binary_classification",
-            "multiclass_classification",
-            "feature_importance",
-            "visualization",
-        ],
-    }
+    tasks = ["classification", "text_classification"]
+    tags = [
+        "sklearn",
+        "binary_classification",
+        "multiclass_classification",
+        "feature_importance",
+        "visualization",
+    ]
     default_params = {
         "kernel_explainer_samples": 10,
         "tree_or_linear_explainer_samples": 200,

validmind/tests/model_validation/sklearn/SilhouettePlot.py

@@ -60,13 +60,11 @@ class SilhouettePlot(Metric):
 
     name = "silhouette_plot"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["clustering"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def run(self):
         y_pred_train = self.inputs.dataset.y_pred(self.inputs.model)

validmind/tests/model_validation/sklearn/TrainingTestDegradation.py

@@ -72,16 +72,14 @@ class TrainingTestDegradation(ThresholdTest):
         "max_threshold": 0.10,  # Maximum 10% degradation
     }
 
-    metadata = {
-        "task_types": ["classification", "text_classification"],
-        "tags": [
-            "sklearn",
-            "binary_classification",
-            "multiclass_classification",
-            "model_performance",
-            "visualization",
-        ],
-    }
+    tasks = ["classification", "text_classification"]
+    tags = [
+        "sklearn",
+        "binary_classification",
+        "multiclass_classification",
+        "model_performance",
+        "visualization",
+    ]
 
     default_metrics = {
         "accuracy": metrics.accuracy_score,

validmind/tests/model_validation/sklearn/VMeasure.py

@@ -50,13 +50,11 @@ class VMeasure(ClusterPerformance):
 
     name = "v_measure_score"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["clustering"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def metric_info(self):
         return {"V Measure": metrics.v_measure_score}

validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py

@@ -85,16 +85,14 @@ class WeakspotsDiagnosis(ThresholdTest):
         },
     }
 
-    metadata = {
-        "task_types": ["classification", "text_classification"],
-        "tags": [
-            "sklearn",
-            "binary_classification",
-            "multiclass_classification",
-            "model_diagnosis",
-            "visualization",
-        ],
-    }
+    tasks = ["classification", "text_classification"]
+    tags = [
+        "sklearn",
+        "binary_classification",
+        "multiclass_classification",
+        "model_diagnosis",
+        "visualization",
+    ]
 
     # TODO: allow configuring
     default_metrics = {

validmind/tests/model_validation/statsmodels/AutoARIMA.py

@@ -58,10 +58,8 @@ class AutoARIMA(Metric):
 
     name = "auto_arima"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data", "forecasting", "model_selection", "statsmodels"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data", "forecasting", "model_selection", "statsmodels"]
 
     max_p = 3
     max_d = 2

validmind/tests/model_validation/statsmodels/BoxPierce.py

@@ -49,10 +49,8 @@ class BoxPierce(Metric):
 
     name = "box_pierce"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data", "forecasting", "statistical_test", "statsmodels"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data", "forecasting", "statistical_test", "statsmodels"]
 
     def run(self):
         """

validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.py

@@ -57,10 +57,9 @@ class CumulativePredictionProbabilities(Metric):
 
     name = "cumulative_prediction_probabilities"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["logistic_regression", "visualization"],
-    }
+    tasks = ["classification"]
+    tags = ["logistic_regression", "visualization"]
+
     default_params = {"title": "Cumulative Probabilities"}
 
     @staticmethod

validmind/tests/model_validation/statsmodels/DurbinWatsonTest.py

@@ -44,10 +44,8 @@ class DurbinWatsonTest(Metric):
 
     name = "durbin_watson"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data", "forecasting", "statistical_test", "statsmodels"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data", "forecasting", "statistical_test", "statsmodels"]
 
     def run(self):
         """

validmind/tests/model_validation/statsmodels/GINITable.py

@@ -59,10 +59,8 @@ class GINITable(Metric):
 
     name = "gini_table"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["visualization", "model_performance"],
-    }
+    tasks = ["classification"]
+    tags = ["visualization", "model_performance"]
 
     def run(self):
 

validmind/tests/model_validation/statsmodels/JarqueBera.py

@@ -45,15 +45,13 @@ class JarqueBera(Metric):
 
     name = "jarque_bera"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["classification", "regression"],
-        "tags": [
-            "tabular_data",
-            "data_distribution",
-            "statistical_test",
-            "statsmodels",
-        ],
-    }
+    tasks = ["classification", "regression"]
+    tags = [
+        "tabular_data",
+        "data_distribution",
+        "statistical_test",
+        "statsmodels",
+    ]
 
     def run(self):
         """

validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py

@@ -52,15 +52,13 @@ class KolmogorovSmirnov(Metric):
     name = "kolmogorov_smirnov"
     required_inputs = ["dataset"]
     default_params = {"dist": "norm"}
-    metadata = {
-        "task_types": ["classification", "regression"],
-        "tags": [
-            "tabular_data",
-            "data_distribution",
-            "statistical_test",
-            "statsmodels",
-        ],
-    }
+    tasks = ["classification", "regression"]
+    tags = [
+        "tabular_data",
+        "data_distribution",
+        "statistical_test",
+        "statsmodels",
+    ]
 
     def summary(self, metric_value):
         results_table = metric_value["metrics_summary"]

validmind/tests/model_validation/statsmodels/LJungBox.py

@@ -45,10 +45,8 @@ class LJungBox(Metric):
 
     name = "ljung_box"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["time_series_data", "forecasting", "statistical_test", "statsmodels"],
-    }
+    tasks = ["regression"]
+    tags = ["time_series_data", "forecasting", "statistical_test", "statsmodels"]
 
     def run(self):
         """

validmind/tests/model_validation/statsmodels/Lilliefors.py

@@ -56,15 +56,13 @@ class Lilliefors(Metric):
 
     name = "lilliefors_test"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["classification", "regression"],
-        "tags": [
-            "tabular_data",
-            "data_distribution",
-            "statistical_test",
-            "statsmodels",
-        ],
-    }
+    tasks = ["classification", "regression"]
+    tags = [
+        "tabular_data",
+        "data_distribution",
+        "statistical_test",
+        "statsmodels",
+    ]
 
     def run(self):
         """

validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.py

@@ -58,10 +58,8 @@ class PredictionProbabilitiesHistogram(Metric):
 
     name = "prediction_probabilities_histogram"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["tabular_data", "visualization", "credit_risk", "logistic_regression"],
-    }
+    tasks = ["classification"]
+    tags = ["tabular_data", "visualization", "credit_risk", "logistic_regression"]
 
     default_params = {"title": "Histogram of Predictive Probabilities"}
 

validmind/tests/model_validation/statsmodels/RegressionCoeffsPlot.py

@@ -55,10 +55,8 @@ class RegressionCoeffsPlot(Metric):
 
     name = "regression_coeffs_plot"
     required_inputs = ["models"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["tabular_data", "visualization", "model_interpretation"],
-    }
+    tasks = ["regression"]
+    tags = ["tabular_data", "visualization", "model_interpretation"]
 
     @staticmethod
     def plot_coefficients_with_ci(model, model_name):

validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py

@@ -57,15 +57,13 @@ class RegressionFeatureSignificance(Metric):
     required_inputs = ["models"]
 
     default_params = {"fontsize": 10, "p_threshold": 0.05}
-    metadata = {
-        "task_types": ["regression"],
-        "tags": [
-            "statistical_test",
-            "model_interpretation",
-            "visualization",
-            "feature_importance",
-        ],
-    }
+    tasks = ["regression"]
+    tags = [
+        "statistical_test",
+        "model_interpretation",
+        "visualization",
+        "feature_importance",
+    ]
 
     def run(self):
         fontsize = self.params["fontsize"]

validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py

@@ -58,10 +58,8 @@ class RegressionModelForecastPlot(Metric):
     name = "regression_forecast_plot"
     required_inputs = ["models", "datasets"]
     default_params = {"start_date": None, "end_date": None}
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["forecasting", "visualization"],
-    }
+    tasks = ["regression"]
+    tags = ["forecasting", "visualization"]
 
     def run(self):
         start_date = self.params["start_date"]

validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py

@@ -64,10 +64,8 @@ class RegressionModelForecastPlotLevels(Metric):
     default_params = {
         "transformation": None,
     }
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["forecasting", "visualization"],
-    }
+    tasks = ["regression"]
+    tags = ["forecasting", "visualization"]
 
     def run(self):
         transformation = self.params["transformation"]

validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py

@@ -62,10 +62,8 @@ class RegressionModelSensitivityPlot(Metric):
         "transformation": None,
         "shocks": [0.1],
     }
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["senstivity_analysis", "visualization"],
-    }
+    tasks = ["regression"]
+    tags = ["senstivity_analysis", "visualization"]
 
     def run(self):
         logger.info(self.params)

validmind/tests/model_validation/statsmodels/RegressionModelSummary.py

@@ -51,10 +51,8 @@ class RegressionModelSummary(Metric):
 
     name = "regression_model_summary"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["model_metadata", "model_comparison"],
-    }
+    tasks = ["regression"]
+    tags = ["model_metadata", "model_comparison"]
 
     def run(self):
         X_columns = self.inputs.dataset.feature_columns

validmind/tests/model_validation/statsmodels/RegressionModelsCoeffs.py

@@ -46,10 +46,8 @@ class RegressionModelsCoeffs(Metric):
 
     name = "regression_models_coefficients"
     required_inputs = ["models"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["model_comparison"],
-    }
+    tasks = ["regression"]
+    tags = ["model_comparison"]
 
     def _build_model_summaries(self, all_coefficients):
         all_models_df = pd.DataFrame()

validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py

@@ -55,14 +55,12 @@ class RegressionPermutationFeatureImportance(Metric):
         "fontsize": 12,
         "figure_height": 500,
     }
-    metadata = {
-        "task_types": ["regression"],
-        "tags": [
-            "statsmodels",
-            "feature_importance",
-            "visualization",
-        ],
-    }
+    tasks = ["regression"]
+    tags = [
+        "statsmodels",
+        "feature_importance",
+        "visualization",
+    ]
 
     def run(self):
         x = self.inputs.dataset.x_df()

validmind/tests/model_validation/statsmodels/RunsTest.py

@@ -50,10 +50,8 @@ class RunsTest(Metric):
 
     name = "runs_test"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["classification", "regression"],
-        "tags": ["tabular_data", "statistical_test", "statsmodels"],
-    }
+    tasks = ["classification", "regression"]
+    tags = ["tabular_data", "statistical_test", "statsmodels"]
 
     def run(self):
         """

validmind/tests/model_validation/statsmodels/ScorecardHistogram.py

@@ -52,10 +52,9 @@ class ScorecardHistogram(Metric):
 
     name = "scorecard_histogram"
     required_inputs = ["datasets"]
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["tabular_data", "visualization", "credit_risk"],
-    }
+    tasks = ["classification"]
+    tags = ["tabular_data", "visualization", "credit_risk"]
+
     default_params = {
         "title": "Histogram of Scores",
         "score_column": "score",

validmind/tests/model_validation/statsmodels/ShapiroWilk.py

@@ -44,10 +44,8 @@ class ShapiroWilk(Metric):
 
     name = "shapiro_wilk"
     required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["classification", "regression"],
-        "tags": ["tabular_data", "data_distribution", "statistical_test"],
-    }
+    tasks = ["classification", "regression"]
+    tags = ["tabular_data", "data_distribution", "statistical_test"]
 
     def run(self):
         """

validmind/tests/prompt_validation/Bias.py

@@ -75,10 +75,8 @@ class Bias(ThresholdTest):
     name = "bias"
     required_inputs = ["model.prompt"]
     default_params = {"min_threshold": 7}
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["llm", "few_shot"],
-    }
+    tasks = ["text_classification", "text_summarization"]
+    tags = ["llm", "few_shot"]
 
     system_prompt = """
 You are a prompt evaluation AI. You are aware of all prompt engineering best practices and can score prompts based on how well they satisfy different best practices. You analyse the prompts step-by-step based on provided documentation and provide a score and an explanation for how you produced that score.

validmind/tests/prompt_validation/Clarity.py

@@ -64,10 +64,8 @@ class Clarity(ThresholdTest):
     name = "clarity"
     required_inputs = ["model.prompt"]
     default_params = {"min_threshold": 7}
-    metadata = {
-        "task_types": ["text_classification", "text_summarization"],
-        "tags": ["llm", "zero_shot", "few_shot"],
-    }
+    tasks = ["text_classification", "text_summarization"]
+    tags = ["llm", "zero_shot", "few_shot"]
 
     system_prompt = """
 You are a prompt evaluation AI. You are aware of all prompt engineering best practices and can score prompts based on how well they satisfy different metrics. You analyse the prompts step-by-step based on provided documentation and provide a score and an explanation for how you produced that score.