validmind 1.11.5__py3-none-any.whl → 1.11.6__py3-none-any.whl

validmind/client.py

@@ -91,9 +91,7 @@ def init_model(
 
     if not Model.is_supported_model(model):
         raise ValueError(
-            "Model type {} is not supported at the moment.".format(
-                Model.model_class(model)
-            )
+            f"Model type {Model.model_library(model)}.{Model.model_class(model)} is not supported at the moment."
         )
 
     return Model.init_vm_model(

validmind/model_validation/model_metadata.py

@@ -18,6 +18,13 @@ SUPPORTED_STATSMODELS_LINK_FUNCTIONS = {
 }
 
 
+def get_catboost_version():
+    if "catboost" in sys.modules:
+        return sys.modules["catboost"].__version__
+
+    return "n/a"
+
+
 def get_pytorch_version():
     if "torch" in sys.modules:
         return sys.modules["torch"].__version__
@@ -113,6 +120,12 @@ def get_info_from_model_instance(model):
         subtask = "binary"
         framework = "PyTorch"
         framework_version = get_pytorch_version()
+    elif model_class == "CatBoostClassifier":
+        architecture = "Gradient Boosting"
+        task = "classification"
+        subtask = "binary"
+        framework = "CatBoost"
+        framework_version = get_catboost_version()
     else:
         raise ValueError(f"Model class {model_class} is not supported by this test")
 
@@ -162,6 +175,8 @@ def get_params_from_model_instance(model):
         params = model.get_params()
     elif model_library == "pytorch":
         params = {}
+    elif model_library == "catboost":
+        params = model.get_all_params()
     else:
         raise ValueError(f"Model library {model_library} is not supported by this test")
 

validmind/model_validation/sklearn/metrics.py

@@ -444,8 +444,12 @@ class SHAPGlobalImportance(Metric):
         # the shap library generates a bunch of annoying warnings that we don't care about
         warnings.filterwarnings("ignore", category=UserWarning)
 
-        # RandomForestClassifier applies here too
-        if model_class == "XGBClassifier" or model_class == "RandomForestClassifier":
+        # Any tree based model can go here
+        if (
+            model_class == "XGBClassifier"
+            or model_class == "RandomForestClassifier"
+            or model_class == "CatBoostClassifier"
+        ):
             explainer = shap.TreeExplainer(trained_model)
         elif (
             model_class == "LogisticRegression"
@@ -485,6 +489,8 @@ class PopulationStabilityIndex(Metric):
             print(f"Skiping PSI for {model_library} models")
             return
 
-        psi_df = _get_psi(self.model.y_train_predict, self.model.y_test_predict)
+        psi_df = _get_psi(
+            self.model.y_train_predict.copy(), self.model.y_test_predict.copy()
+        )
 
         return self.cache_results(metric_value=psi_df)

validmind/model_validation/sklearn/threshold_tests.py

@@ -318,9 +318,13 @@ class OverfitDiagnosis(ThresholdTest):
             features_list = self.params["features_columns"]
 
         # Check if all elements from features_list are present in the feature columns
-        all_present = all(elem in self.model.train_ds.get_features_columns() for elem in features_list)
+        all_present = all(
+            elem in self.model.train_ds.get_features_columns() for elem in features_list
+        )
         if not all_present:
-            raise ValueError("The list of feature columns provided do not match with training dataset feature columns")
+            raise ValueError(
+                "The list of feature columns provided do not match with training dataset feature columns"
+            )
 
         if not isinstance(features_list, list):
             raise ValueError(
@@ -595,10 +599,14 @@ class WeakspotsDiagnosis(ThresholdTest):
             features_list = self.params["features_columns"]
 
         # Check if all elements from features_list are present in the feature columns
-        all_present = all(elem in self.model.train_ds.get_features_columns() for elem in features_list)
+        all_present = all(
+            elem in self.model.train_ds.get_features_columns() for elem in features_list
+        )
         if not all_present:
-            raise ValueError("The list of feature columns provided do not match with "
-                             + "training dataset feature columns")
+            raise ValueError(
+                "The list of feature columns provided do not match with "
+                + "training dataset feature columns"
+            )
 
         target_column = self.model.train_ds.target_column
         prediction_column = f"{target_column}_pred"
@@ -866,14 +874,20 @@ class RobustnessDiagnosis(ThresholdTest):
             features_list = self.model.train_ds.get_numeric_features_columns()
 
         # Check if all elements from features_list are present in the numerical feature columns
-        all_present = all(elem in self.model.train_ds.get_numeric_features_columns()
-                          for elem in features_list)
+        all_present = all(
+            elem in self.model.train_ds.get_numeric_features_columns()
+            for elem in features_list
+        )
         if not all_present:
-            raise ValueError("The list of feature columns provided do not match with training "
-                             + "dataset numerical feature columns")
+            raise ValueError(
+                "The list of feature columns provided do not match with training "
+                + "dataset numerical feature columns"
+            )
 
         # Remove target column if it exist in the list
-        features_list = [col for col in features_list if col != self.model.train_ds.target_column]
+        features_list = [
+            col for col in features_list if col != self.model.train_ds.target_column
+        ]
 
         train_df = self.model.train_ds.x.copy()
         train_y_true = self.model.train_ds.y
@@ -884,7 +898,9 @@ class RobustnessDiagnosis(ThresholdTest):
         test_results = []
         test_figures = []
 
-        results_headers = ["Perturbation Size", "Dataset Type", "Records"] + list(self.default_metrics.keys())
+        results_headers = ["Perturbation Size", "Dataset Type", "Records"] + list(
+            self.default_metrics.keys()
+        )
         results = {k: [] for k in results_headers}
 
         # Iterate scaling factor for the standard deviation list
@@ -920,14 +936,20 @@ class RobustnessDiagnosis(ThresholdTest):
             )
         )
 
-        train_acc = df.loc[(df['Dataset Type'] == "Training") , 'accuracy'].values[0]
-        test_acc = df.loc[(df['Dataset Type'] == "Test") , 'accuracy'].values[0]
-
-        df["Passed"] = np.where((df['Dataset Type'] == "Training") & (df['accuracy'] >= (train_acc - accuracy_threshold)),
-                                True,
-                                np.where((df['Dataset Type'] == "Test") & (df['accuracy'] >= (test_acc - accuracy_threshold)),
-                                         True,
-                                         False))
+        train_acc = df.loc[(df["Dataset Type"] == "Training"), "accuracy"].values[0]
+        test_acc = df.loc[(df["Dataset Type"] == "Test"), "accuracy"].values[0]
+
+        df["Passed"] = np.where(
+            (df["Dataset Type"] == "Training")
+            & (df["accuracy"] >= (train_acc - accuracy_threshold)),
+            True,
+            np.where(
+                (df["Dataset Type"] == "Test")
+                & (df["accuracy"] >= (test_acc - accuracy_threshold)),
+                True,
+                False,
+            ),
+        )
         test_results.append(
             TestResult(
                 test_name="accuracy",
@@ -936,7 +958,9 @@ class RobustnessDiagnosis(ThresholdTest):
                 values=df.to_dict(),
             )
         )
-        return self.cache_results(test_results, passed=df['Passed'].all(), figures=test_figures)
+        return self.cache_results(
+            test_results, passed=df["Passed"].all(), figures=test_figures
+        )
 
     def _compute_metrics(
         self,

validmind/vm_models/model.py

@@ -9,6 +9,7 @@ from .dataset import Dataset
 # import torch.nn as nn
 
 SUPPORTED_MODEL_TYPES = [
+    "catboost.CatBoostClassifier",
     "pytorch.PyTorchModel",
     "sklearn.LogisticRegression",
     "sklearn.LinearRegression",

{validmind-1.11.5.dist-info → validmind-1.11.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 1.11.5
+Version: 1.11.6
 Summary: ValidMind Developer Framework
 Author: Andres Rodriguez
 Author-email: andres@validmind.ai
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Provides-Extra: r-support
 Requires-Dist: arch (>=5.4.0,<6.0.0)
+Requires-Dist: catboost (>=1.2,<2.0)
 Requires-Dist: click (>=8.0.4,<9.0.0)
 Requires-Dist: dython (>=0.7.1,<0.8.0)
 Requires-Dist: ipython (==7.34.0)

{validmind-1.11.5.dist-info → validmind-1.11.6.dist-info}/RECORD

@@ -1,6 +1,6 @@
 validmind/__init__.py,sha256=ND6a4RZrm1QFdYvDnGqhfwoXFsC762CJrUzdPdcVtp0,1443
 validmind/api_client.py,sha256=bjRsGzSJlviQVah8Dben_NHnaYcoMov06xeT2rxZiG0,11540
-validmind/client.py,sha256=z2bS0oyBz5pTf0II6YLy3HlIlGjXpt_QyETBqAkOmPI,9232
+validmind/client.py,sha256=hUdQGgKL_aJDW_rRQufQoOGeSIroUzmy8S3xCDaKL44,9223
 validmind/data_validation/__init__.py,sha256=xytRpsfQ86fDnIZRoAO7GMVVU_TwWVMXxSCwm0mb45I,590
 validmind/data_validation/metrics.py,sha256=fLi9vkalf8Yp1MQ0HyJ7y_RbArf-NriE0-gzeXxlI3Q,44383
 validmind/data_validation/threshold_tests.py,sha256=lUW3_LKHaLg9_Npp-HViOW4cbGbPF18qtprODjQEiEw,31630
@@ -27,10 +27,10 @@ validmind/datasets/regression/models/fred_loan_rates_model_4.pkl,sha256=cSxhpcrI
 validmind/datasets/regression/models/fred_loan_rates_model_5.pkl,sha256=FkNLHq9xkPMbYks_vyMjFL371mw9SQYbP1iX9lY4Ljo,60343
 validmind/model_utils.py,sha256=DuPN2tF582ho167mfodODseJ6LIVuQOF9Xx0zv6b4Yk,10529
 validmind/model_validation/__init__.py,sha256=fYWK9BES5uKGW4yhRLtXk6L2IAytk9xkB-NFVMVK_Gk,43
-validmind/model_validation/model_metadata.py,sha256=Akm6jO9MJN4ybIAV9TivZ-QyuMjewF_4Pg05Y2ND0AY,6712
+validmind/model_validation/model_metadata.py,sha256=r-MyR1KTg7Dnmtbw1VW18dGwDt2mDjjxzk2kqLoOfrw,7153
 validmind/model_validation/sklearn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-validmind/model_validation/sklearn/metrics.py,sha256=JXuYlkJ72qaLzNVeGFmzlLmSQCs2pqWRwopBzcAxn-8,15135
-validmind/model_validation/sklearn/threshold_tests.py,sha256=fU3vy-_RcV0XqNoiwNq0wSWeACBLMsHmsAbXEwgtySM,39678
+validmind/model_validation/sklearn/metrics.py,sha256=J6vizOm06VoMK1YA--3BrBC-JcmcBoC2s7PvHLeRcWY,15251
+validmind/model_validation/sklearn/threshold_tests.py,sha256=Phc4Ppek25Fb5B5bul7X8Zuue_8KU5O1jkAh2Foalsk,39850
 validmind/model_validation/statsmodels/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 validmind/model_validation/statsmodels/metrics.py,sha256=4OUd-pv7kfrnIMXUyYudJX9tPGLH5yELLDBoJ8g5quU,28272
 validmind/model_validation/statsmodels/threshold_tests.py,sha256=SyEv7oaNIgiQ1ML3dQrGN54pSP88j9htOy4V2l4Ipwk,1317
@@ -50,7 +50,7 @@ validmind/vm_models/dataset_utils.py,sha256=xWq6dbxe9fCUdpKCiEFvbxhy9t7cOKh0boS-
 validmind/vm_models/figure.py,sha256=EarfMm49J2BWf6UhsMFGr3VSdGu8v2ZewzArKKp_LfU,592
 validmind/vm_models/metric.py,sha256=kf1N646Wq2pSZ1GkxG1tDoFyqGntN1N-PTMLPz1j-5o,3746
 validmind/vm_models/metric_result.py,sha256=NVPNIt4S9HN05CnFr-DPYp2ZUYoFGVP-zT2KX6gNypQ,1757
-validmind/vm_models/model.py,sha256=segtsvR0fg34hH8qlF9V9ujIVv3VI4s9CBtRX4TVqnE,6530
+validmind/vm_models/model.py,sha256=9ubRdS8eVZaQWtmupLedtD7xf905r3UxePBJrMoxcFs,6565
 validmind/vm_models/plot_utils.py,sha256=BWYc9SwITwZxaAJePVYDLrBSV1XEV0EmVllq9A1K_IM,3721
 validmind/vm_models/result_summary.py,sha256=1YGlWqcO4PY2NtDm7fbuOSLXPIu6KM1USp_pgh6rmJM,1518
 validmind/vm_models/test_context.py,sha256=nWiTIORPNEl7mKkYme8n2QaJyAotp034A2NK33MNmhE,2501
@@ -59,7 +59,7 @@ validmind/vm_models/test_plan_result.py,sha256=oPEkss0eJAetRzv11P4McF9b651v6Tq7k
 validmind/vm_models/test_result.py,sha256=jiX8yb1NptBXZQ_pOyHloc8I6yS6zamYm0j9OWUqrHs,1698
 validmind/vm_models/test_suite.py,sha256=d2yBuLD4ga7Bb_yK67LJ14C58_Mj4GYBPdy4BuxoBog,5407
 validmind/vm_models/threshold_test.py,sha256=F0s4JPN8JXpGjsNU40xaeOgt8kB2BI1GrOrcYCGcxqA,4231
-validmind-1.11.5.dist-info/LICENSE,sha256=oyp_7jnk_p7ZNF9mcWpiHadwWc1JqR1aaemjjfCFscE,5458
-validmind-1.11.5.dist-info/METADATA,sha256=r6WtQH3p_8SwS138CHzzzESPlim0gWxMJf7ctVXKrlg,1420
-validmind-1.11.5.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-validmind-1.11.5.dist-info/RECORD,,
+validmind-1.11.6.dist-info/LICENSE,sha256=oyp_7jnk_p7ZNF9mcWpiHadwWc1JqR1aaemjjfCFscE,5458
+validmind-1.11.6.dist-info/METADATA,sha256=Qbl-H0aQ0ma-SA8SiC7Os1_F94fWMpmhWItRK_k8zXU,1457
+validmind-1.11.6.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+validmind-1.11.6.dist-info/RECORD,,