validmind 1.11.4__py3-none-any.whl → 1.11.6__py3-none-any.whl

validmind/client.py

@@ -91,9 +91,7 @@ def init_model(
 
     if not Model.is_supported_model(model):
         raise ValueError(
-            "Model type {} is not supported at the moment.".format(
-                Model.model_class(model)
-            )
+            f"Model type {Model.model_library(model)}.{Model.model_class(model)} is not supported at the moment."
         )
 
     return Model.init_vm_model(

validmind/model_validation/model_metadata.py

@@ -18,6 +18,13 @@ SUPPORTED_STATSMODELS_LINK_FUNCTIONS = {
 }
 
 
+def get_catboost_version():
+    if "catboost" in sys.modules:
+        return sys.modules["catboost"].__version__
+
+    return "n/a"
+
+
 def get_pytorch_version():
     if "torch" in sys.modules:
         return sys.modules["torch"].__version__
@@ -113,6 +120,12 @@ def get_info_from_model_instance(model):
         subtask = "binary"
         framework = "PyTorch"
         framework_version = get_pytorch_version()
+    elif model_class == "CatBoostClassifier":
+        architecture = "Gradient Boosting"
+        task = "classification"
+        subtask = "binary"
+        framework = "CatBoost"
+        framework_version = get_catboost_version()
     else:
         raise ValueError(f"Model class {model_class} is not supported by this test")
 
@@ -162,6 +175,8 @@ def get_params_from_model_instance(model):
         params = model.get_params()
     elif model_library == "pytorch":
         params = {}
+    elif model_library == "catboost":
+        params = model.get_all_params()
     else:
         raise ValueError(f"Model library {model_library} is not supported by this test")
 

validmind/model_validation/sklearn/metrics.py

@@ -444,8 +444,12 @@ class SHAPGlobalImportance(Metric):
         # the shap library generates a bunch of annoying warnings that we don't care about
         warnings.filterwarnings("ignore", category=UserWarning)
 
-        # RandomForestClassifier applies here too
-        if model_class == "XGBClassifier" or model_class == "RandomForestClassifier":
+        # Any tree based model can go here
+        if (
+            model_class == "XGBClassifier"
+            or model_class == "RandomForestClassifier"
+            or model_class == "CatBoostClassifier"
+        ):
             explainer = shap.TreeExplainer(trained_model)
         elif (
             model_class == "LogisticRegression"
@@ -485,6 +489,8 @@ class PopulationStabilityIndex(Metric):
             print(f"Skiping PSI for {model_library} models")
             return
 
-        psi_df = _get_psi(self.model.y_train_predict, self.model.y_test_predict)
+        psi_df = _get_psi(
+            self.model.y_train_predict.copy(), self.model.y_test_predict.copy()
+        )
 
         return self.cache_results(metric_value=psi_df)

validmind/model_validation/sklearn/threshold_tests.py

@@ -313,13 +313,19 @@ class OverfitDiagnosis(ThresholdTest):
             raise ValueError("model must of provided to run this test")
 
         if self.params["features_columns"] is None:
-            features_list = [
-                field_dict["id"] for field_dict in self.model.train_ds.fields
-            ]
-            features_list.remove(self.model.train_ds.target_column)
+            features_list = self.model.train_ds.get_features_columns()
         else:
             features_list = self.params["features_columns"]
 
+        # Check if all elements from features_list are present in the feature columns
+        all_present = all(
+            elem in self.model.train_ds.get_features_columns() for elem in features_list
+        )
+        if not all_present:
+            raise ValueError(
+                "The list of feature columns provided do not match with training dataset feature columns"
+            )
+
         if not isinstance(features_list, list):
             raise ValueError(
                 "features_columns must be a list of features you would like to test"
@@ -344,7 +350,11 @@ class OverfitDiagnosis(ThresholdTest):
         results_headers.extend(self.default_metrics.keys())
 
         for feature_column in features_list:
-            train_df["bin"] = pd.cut(train_df[feature_column], bins=10)
+            bins = 10
+            if feature_column in self.model.train_ds.get_categorical_features_columns():
+                bins = len(train_df[feature_column].unique())
+            train_df["bin"] = pd.cut(train_df[feature_column], bins=bins)
+
             results_train = {k: [] for k in results_headers}
             results_test = {k: [] for k in results_headers}
 
@@ -583,17 +593,21 @@ class WeakspotsDiagnosis(ThresholdTest):
         if self.model is None:
             raise ValueError("model must of provided to run this test")
 
-        if "features_columns" not in self.params:
-            raise ValueError("features_columns must be provided in params")
-
         if self.params["features_columns"] is None:
-            features_list = [
-                field_dict["id"] for field_dict in self.model.train_ds.fields
-            ]
-            features_list.remove(self.model.train_ds.target_column)
+            features_list = self.model.train_ds.get_features_columns()
         else:
             features_list = self.params["features_columns"]
 
+        # Check if all elements from features_list are present in the feature columns
+        all_present = all(
+            elem in self.model.train_ds.get_features_columns() for elem in features_list
+        )
+        if not all_present:
+            raise ValueError(
+                "The list of feature columns provided do not match with "
+                + "training dataset feature columns"
+            )
+
         target_column = self.model.train_ds.target_column
         prediction_column = f"{target_column}_pred"
 
@@ -610,7 +624,11 @@ class WeakspotsDiagnosis(ThresholdTest):
         results_headers = ["slice", "shape"]
         results_headers.extend(self.default_metrics.keys())
         for feature in features_list:
-            train_df["bin"] = pd.cut(train_df[feature], bins=10)
+            bins = 10
+            if feature in self.model.train_ds.get_categorical_features_columns():
+                bins = len(train_df[feature].unique())
+            train_df["bin"] = pd.cut(train_df[feature], bins=bins)
+
             results_train = {k: [] for k in results_headers}
             results_test = {k: [] for k in results_headers}
 
@@ -811,6 +829,7 @@ class RobustnessDiagnosis(ThresholdTest):
     default_params = {
         "features_columns": None,
         "scaling_factor_std_dev_list": [0.01, 0.02],
+        "accuracy_decay_threshold": 3,
     }
     default_metrics = {
         "accuracy": metrics.accuracy_score,
@@ -839,6 +858,10 @@ class RobustnessDiagnosis(ThresholdTest):
             raise ValueError("scaling_factor_std_dev_list must be provided in params")
         x_std_dev_list = self.params["scaling_factor_std_dev_list"]
 
+        if self.params["accuracy_decay_threshold"] is None:
+            raise ValueError("accuracy_decay_threshold must be provided in params")
+        accuracy_threshold = self.params["accuracy_decay_threshold"]
+
         if self.model is None:
             raise ValueError("model must of provided to run this test")
 
@@ -846,20 +869,25 @@ class RobustnessDiagnosis(ThresholdTest):
         if "features_columns" not in self.params:
             raise ValueError("features_columns must be provided in params")
 
-        # Identify numeric features
-        numeric_features_columns = [
-            field_dic["id"]
-            for field_dic in self.model.train_ds.fields
-            if field_dic["type"] == "Numeric"
-        ]
-        if self.params["features_columns"] is None:
-            features_list = numeric_features_columns
-        else:
-            features_list = self.params["features_columns"]
+        features_list = self.params["features_columns"]
+        if features_list is None:
+            features_list = self.model.train_ds.get_numeric_features_columns()
+
+        # Check if all elements from features_list are present in the numerical feature columns
+        all_present = all(
+            elem in self.model.train_ds.get_numeric_features_columns()
+            for elem in features_list
+        )
+        if not all_present:
+            raise ValueError(
+                "The list of feature columns provided do not match with training "
+                + "dataset numerical feature columns"
+            )
 
         # Remove target column if it exist in the list
-        if self.model.train_ds.target_column in features_list:
-            features_list.remove(self.model.train_ds.target_column)
+        features_list = [
+            col for col in features_list if col != self.model.train_ds.target_column
+        ]
 
         train_df = self.model.train_ds.x.copy()
         train_y_true = self.model.train_ds.y
@@ -870,8 +898,9 @@ class RobustnessDiagnosis(ThresholdTest):
         test_results = []
         test_figures = []
 
-        results_headers = ["Perturbation Size", "Dataset Type", "Records"]
-        results_headers.extend(self.default_metrics.keys())
+        results_headers = ["Perturbation Size", "Dataset Type", "Records"] + list(
+            self.default_metrics.keys()
+        )
         results = {k: [] for k in results_headers}
 
         # Iterate scaling factor for the standard deviation list
@@ -881,10 +910,10 @@ class RobustnessDiagnosis(ThresholdTest):
 
             # Add noise to numeric features columns provided by user
             for feature in features_list:
-                temp_train_df[feature] = self.add_noise_std_dev(
+                temp_train_df[feature] = self._add_noise_std_dev(
                     temp_train_df[feature].to_list(), x_std_dev
                 )
-                temp_test_df[feature] = self.add_noise_std_dev(
+                temp_test_df[feature] = self._add_noise_std_dev(
                     temp_test_df[feature].to_list(), x_std_dev
                 )
 
@@ -907,15 +936,31 @@ class RobustnessDiagnosis(ThresholdTest):
             )
         )
 
+        train_acc = df.loc[(df["Dataset Type"] == "Training"), "accuracy"].values[0]
+        test_acc = df.loc[(df["Dataset Type"] == "Test"), "accuracy"].values[0]
+
+        df["Passed"] = np.where(
+            (df["Dataset Type"] == "Training")
+            & (df["accuracy"] >= (train_acc - accuracy_threshold)),
+            True,
+            np.where(
+                (df["Dataset Type"] == "Test")
+                & (df["accuracy"] >= (test_acc - accuracy_threshold)),
+                True,
+                False,
+            ),
+        )
         test_results.append(
             TestResult(
                 test_name="accuracy",
-                column=features_list[0],
+                column=features_list,
                 passed=True,
-                values=df.to_dict(orient="list"),
+                values=df.to_dict(),
             )
         )
-        return self.cache_results(test_results, passed=True, figures=test_figures)
+        return self.cache_results(
+            test_results, passed=df["Passed"].all(), figures=test_figures
+        )
 
     def _compute_metrics(
         self,
@@ -946,7 +991,7 @@ class RobustnessDiagnosis(ThresholdTest):
         for metric, metric_fn in self.default_metrics.items():
             results[metric].append(metric_fn(y_true.values, y_prediction) * 100)
 
-    def add_noise_std_dev(
+    def _add_noise_std_dev(
         self, values: List[float], x_std_dev: float
     ) -> Tuple[List[float], float]:
         """

validmind/vm_models/model.py

@@ -9,6 +9,7 @@ from .dataset import Dataset
 # import torch.nn as nn
 
 SUPPORTED_MODEL_TYPES = [
+    "catboost.CatBoostClassifier",
     "pytorch.PyTorchModel",
     "sklearn.LogisticRegression",
     "sklearn.LinearRegression",

{validmind-1.11.4.dist-info → validmind-1.11.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 1.11.4
+Version: 1.11.6
 Summary: ValidMind Developer Framework
 Author: Andres Rodriguez
 Author-email: andres@validmind.ai
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Provides-Extra: r-support
 Requires-Dist: arch (>=5.4.0,<6.0.0)
+Requires-Dist: catboost (>=1.2,<2.0)
 Requires-Dist: click (>=8.0.4,<9.0.0)
 Requires-Dist: dython (>=0.7.1,<0.8.0)
 Requires-Dist: ipython (==7.34.0)

{validmind-1.11.4.dist-info → validmind-1.11.6.dist-info}/RECORD

@@ -1,6 +1,6 @@
 validmind/__init__.py,sha256=ND6a4RZrm1QFdYvDnGqhfwoXFsC762CJrUzdPdcVtp0,1443
 validmind/api_client.py,sha256=bjRsGzSJlviQVah8Dben_NHnaYcoMov06xeT2rxZiG0,11540
-validmind/client.py,sha256=z2bS0oyBz5pTf0II6YLy3HlIlGjXpt_QyETBqAkOmPI,9232
+validmind/client.py,sha256=hUdQGgKL_aJDW_rRQufQoOGeSIroUzmy8S3xCDaKL44,9223
 validmind/data_validation/__init__.py,sha256=xytRpsfQ86fDnIZRoAO7GMVVU_TwWVMXxSCwm0mb45I,590
 validmind/data_validation/metrics.py,sha256=fLi9vkalf8Yp1MQ0HyJ7y_RbArf-NriE0-gzeXxlI3Q,44383
 validmind/data_validation/threshold_tests.py,sha256=lUW3_LKHaLg9_Npp-HViOW4cbGbPF18qtprODjQEiEw,31630
@@ -27,10 +27,10 @@ validmind/datasets/regression/models/fred_loan_rates_model_4.pkl,sha256=cSxhpcrI
 validmind/datasets/regression/models/fred_loan_rates_model_5.pkl,sha256=FkNLHq9xkPMbYks_vyMjFL371mw9SQYbP1iX9lY4Ljo,60343
 validmind/model_utils.py,sha256=DuPN2tF582ho167mfodODseJ6LIVuQOF9Xx0zv6b4Yk,10529
 validmind/model_validation/__init__.py,sha256=fYWK9BES5uKGW4yhRLtXk6L2IAytk9xkB-NFVMVK_Gk,43
-validmind/model_validation/model_metadata.py,sha256=Akm6jO9MJN4ybIAV9TivZ-QyuMjewF_4Pg05Y2ND0AY,6712
+validmind/model_validation/model_metadata.py,sha256=r-MyR1KTg7Dnmtbw1VW18dGwDt2mDjjxzk2kqLoOfrw,7153
 validmind/model_validation/sklearn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-validmind/model_validation/sklearn/metrics.py,sha256=JXuYlkJ72qaLzNVeGFmzlLmSQCs2pqWRwopBzcAxn-8,15135
-validmind/model_validation/sklearn/threshold_tests.py,sha256=9bhXeM8tI6e7gOHFZrhG6vT5tCaMkVCQO0JTM33fiKk,38039
+validmind/model_validation/sklearn/metrics.py,sha256=J6vizOm06VoMK1YA--3BrBC-JcmcBoC2s7PvHLeRcWY,15251
+validmind/model_validation/sklearn/threshold_tests.py,sha256=Phc4Ppek25Fb5B5bul7X8Zuue_8KU5O1jkAh2Foalsk,39850
 validmind/model_validation/statsmodels/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 validmind/model_validation/statsmodels/metrics.py,sha256=4OUd-pv7kfrnIMXUyYudJX9tPGLH5yELLDBoJ8g5quU,28272
 validmind/model_validation/statsmodels/threshold_tests.py,sha256=SyEv7oaNIgiQ1ML3dQrGN54pSP88j9htOy4V2l4Ipwk,1317
@@ -50,7 +50,7 @@ validmind/vm_models/dataset_utils.py,sha256=xWq6dbxe9fCUdpKCiEFvbxhy9t7cOKh0boS-
 validmind/vm_models/figure.py,sha256=EarfMm49J2BWf6UhsMFGr3VSdGu8v2ZewzArKKp_LfU,592
 validmind/vm_models/metric.py,sha256=kf1N646Wq2pSZ1GkxG1tDoFyqGntN1N-PTMLPz1j-5o,3746
 validmind/vm_models/metric_result.py,sha256=NVPNIt4S9HN05CnFr-DPYp2ZUYoFGVP-zT2KX6gNypQ,1757
-validmind/vm_models/model.py,sha256=segtsvR0fg34hH8qlF9V9ujIVv3VI4s9CBtRX4TVqnE,6530
+validmind/vm_models/model.py,sha256=9ubRdS8eVZaQWtmupLedtD7xf905r3UxePBJrMoxcFs,6565
 validmind/vm_models/plot_utils.py,sha256=BWYc9SwITwZxaAJePVYDLrBSV1XEV0EmVllq9A1K_IM,3721
 validmind/vm_models/result_summary.py,sha256=1YGlWqcO4PY2NtDm7fbuOSLXPIu6KM1USp_pgh6rmJM,1518
 validmind/vm_models/test_context.py,sha256=nWiTIORPNEl7mKkYme8n2QaJyAotp034A2NK33MNmhE,2501
@@ -59,7 +59,7 @@ validmind/vm_models/test_plan_result.py,sha256=oPEkss0eJAetRzv11P4McF9b651v6Tq7k
 validmind/vm_models/test_result.py,sha256=jiX8yb1NptBXZQ_pOyHloc8I6yS6zamYm0j9OWUqrHs,1698
 validmind/vm_models/test_suite.py,sha256=d2yBuLD4ga7Bb_yK67LJ14C58_Mj4GYBPdy4BuxoBog,5407
 validmind/vm_models/threshold_test.py,sha256=F0s4JPN8JXpGjsNU40xaeOgt8kB2BI1GrOrcYCGcxqA,4231
-validmind-1.11.4.dist-info/LICENSE,sha256=oyp_7jnk_p7ZNF9mcWpiHadwWc1JqR1aaemjjfCFscE,5458
-validmind-1.11.4.dist-info/METADATA,sha256=3lWj-Z2EqJXXc2LlEwgJtTpLZGUW7K71hs3xG_CokkE,1420
-validmind-1.11.4.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-validmind-1.11.4.dist-info/RECORD,,
+validmind-1.11.6.dist-info/LICENSE,sha256=oyp_7jnk_p7ZNF9mcWpiHadwWc1JqR1aaemjjfCFscE,5458
+validmind-1.11.6.dist-info/METADATA,sha256=Qbl-H0aQ0ma-SA8SiC7Os1_F94fWMpmhWItRK_k8zXU,1457
+validmind-1.11.6.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+validmind-1.11.6.dist-info/RECORD,,