validmind 1.11.4__tar.gz → 1.11.6__tar.gz

{validmind-1.11.4 → validmind-1.11.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 1.11.4
+Version: 1.11.6
 Summary: ValidMind Developer Framework
 Author: Andres Rodriguez
 Author-email: andres@validmind.ai
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Provides-Extra: r-support
 Requires-Dist: arch (>=5.4.0,<6.0.0)
+Requires-Dist: catboost (>=1.2,<2.0)
 Requires-Dist: click (>=8.0.4,<9.0.0)
 Requires-Dist: dython (>=0.7.1,<0.8.0)
 Requires-Dist: ipython (==7.34.0)

{validmind-1.11.4 → validmind-1.11.6}/pyproject.toml

@@ -13,7 +13,7 @@ description = "ValidMind Developer Framework"
 #   "validmind/**/*.so",
 # ]
 name = "validmind"
-version = "1.11.4"
+version = "1.11.6"
 
 [tool.poetry.dependencies]
 arch = "^5.4.0"
@@ -41,6 +41,7 @@ tabulate = "^0.8.9"
 tqdm = "^4.64.0"
 xgboost = "^1.5.2"
 markdown = "^3.4.3"
+catboost = "^1.2"
 
 [tool.poetry.group.dev.dependencies]
 black = "^22.1.0"

{validmind-1.11.4 → validmind-1.11.6}/validmind/client.py

@@ -91,9 +91,7 @@ def init_model(
 
     if not Model.is_supported_model(model):
         raise ValueError(
-            "Model type {} is not supported at the moment.".format(
-                Model.model_class(model)
-            )
+            f"Model type {Model.model_library(model)}.{Model.model_class(model)} is not supported at the moment."
         )
 
     return Model.init_vm_model(

{validmind-1.11.4 → validmind-1.11.6}/validmind/model_validation/model_metadata.py

@@ -18,6 +18,13 @@ SUPPORTED_STATSMODELS_LINK_FUNCTIONS = {
 }
 
 
+def get_catboost_version():
+    if "catboost" in sys.modules:
+        return sys.modules["catboost"].__version__
+
+    return "n/a"
+
+
 def get_pytorch_version():
     if "torch" in sys.modules:
         return sys.modules["torch"].__version__
@@ -113,6 +120,12 @@ def get_info_from_model_instance(model):
         subtask = "binary"
         framework = "PyTorch"
         framework_version = get_pytorch_version()
+    elif model_class == "CatBoostClassifier":
+        architecture = "Gradient Boosting"
+        task = "classification"
+        subtask = "binary"
+        framework = "CatBoost"
+        framework_version = get_catboost_version()
     else:
         raise ValueError(f"Model class {model_class} is not supported by this test")
 
@@ -162,6 +175,8 @@ def get_params_from_model_instance(model):
         params = model.get_params()
     elif model_library == "pytorch":
         params = {}
+    elif model_library == "catboost":
+        params = model.get_all_params()
     else:
         raise ValueError(f"Model library {model_library} is not supported by this test")
 

{validmind-1.11.4 → validmind-1.11.6}/validmind/model_validation/sklearn/metrics.py

@@ -444,8 +444,12 @@ class SHAPGlobalImportance(Metric):
         # the shap library generates a bunch of annoying warnings that we don't care about
         warnings.filterwarnings("ignore", category=UserWarning)
 
-        # RandomForestClassifier applies here too
-        if model_class == "XGBClassifier" or model_class == "RandomForestClassifier":
+        # Any tree based model can go here
+        if (
+            model_class == "XGBClassifier"
+            or model_class == "RandomForestClassifier"
+            or model_class == "CatBoostClassifier"
+        ):
             explainer = shap.TreeExplainer(trained_model)
         elif (
             model_class == "LogisticRegression"
@@ -485,6 +489,8 @@ class PopulationStabilityIndex(Metric):
             print(f"Skiping PSI for {model_library} models")
             return
 
-        psi_df = _get_psi(self.model.y_train_predict, self.model.y_test_predict)
+        psi_df = _get_psi(
+            self.model.y_train_predict.copy(), self.model.y_test_predict.copy()
+        )
 
         return self.cache_results(metric_value=psi_df)

{validmind-1.11.4 → validmind-1.11.6}/validmind/model_validation/sklearn/threshold_tests.py

@@ -313,13 +313,19 @@ class OverfitDiagnosis(ThresholdTest):
             raise ValueError("model must of provided to run this test")
 
         if self.params["features_columns"] is None:
-            features_list = [
-                field_dict["id"] for field_dict in self.model.train_ds.fields
-            ]
-            features_list.remove(self.model.train_ds.target_column)
+            features_list = self.model.train_ds.get_features_columns()
         else:
             features_list = self.params["features_columns"]
 
+        # Check if all elements from features_list are present in the feature columns
+        all_present = all(
+            elem in self.model.train_ds.get_features_columns() for elem in features_list
+        )
+        if not all_present:
+            raise ValueError(
+                "The list of feature columns provided do not match with training dataset feature columns"
+            )
+
         if not isinstance(features_list, list):
             raise ValueError(
                 "features_columns must be a list of features you would like to test"
@@ -344,7 +350,11 @@ class OverfitDiagnosis(ThresholdTest):
         results_headers.extend(self.default_metrics.keys())
 
         for feature_column in features_list:
-            train_df["bin"] = pd.cut(train_df[feature_column], bins=10)
+            bins = 10
+            if feature_column in self.model.train_ds.get_categorical_features_columns():
+                bins = len(train_df[feature_column].unique())
+            train_df["bin"] = pd.cut(train_df[feature_column], bins=bins)
+
             results_train = {k: [] for k in results_headers}
             results_test = {k: [] for k in results_headers}
 
@@ -583,17 +593,21 @@ class WeakspotsDiagnosis(ThresholdTest):
         if self.model is None:
             raise ValueError("model must of provided to run this test")
 
-        if "features_columns" not in self.params:
-            raise ValueError("features_columns must be provided in params")
-
         if self.params["features_columns"] is None:
-            features_list = [
-                field_dict["id"] for field_dict in self.model.train_ds.fields
-            ]
-            features_list.remove(self.model.train_ds.target_column)
+            features_list = self.model.train_ds.get_features_columns()
         else:
             features_list = self.params["features_columns"]
 
+        # Check if all elements from features_list are present in the feature columns
+        all_present = all(
+            elem in self.model.train_ds.get_features_columns() for elem in features_list
+        )
+        if not all_present:
+            raise ValueError(
+                "The list of feature columns provided do not match with "
+                + "training dataset feature columns"
+            )
+
         target_column = self.model.train_ds.target_column
         prediction_column = f"{target_column}_pred"
 
@@ -610,7 +624,11 @@ class WeakspotsDiagnosis(ThresholdTest):
         results_headers = ["slice", "shape"]
         results_headers.extend(self.default_metrics.keys())
         for feature in features_list:
-            train_df["bin"] = pd.cut(train_df[feature], bins=10)
+            bins = 10
+            if feature in self.model.train_ds.get_categorical_features_columns():
+                bins = len(train_df[feature].unique())
+            train_df["bin"] = pd.cut(train_df[feature], bins=bins)
+
             results_train = {k: [] for k in results_headers}
             results_test = {k: [] for k in results_headers}
 
@@ -811,6 +829,7 @@ class RobustnessDiagnosis(ThresholdTest):
     default_params = {
         "features_columns": None,
         "scaling_factor_std_dev_list": [0.01, 0.02],
+        "accuracy_decay_threshold": 3,
     }
     default_metrics = {
         "accuracy": metrics.accuracy_score,
@@ -839,6 +858,10 @@ class RobustnessDiagnosis(ThresholdTest):
             raise ValueError("scaling_factor_std_dev_list must be provided in params")
         x_std_dev_list = self.params["scaling_factor_std_dev_list"]
 
+        if self.params["accuracy_decay_threshold"] is None:
+            raise ValueError("accuracy_decay_threshold must be provided in params")
+        accuracy_threshold = self.params["accuracy_decay_threshold"]
+
         if self.model is None:
             raise ValueError("model must of provided to run this test")
 
@@ -846,20 +869,25 @@ class RobustnessDiagnosis(ThresholdTest):
         if "features_columns" not in self.params:
             raise ValueError("features_columns must be provided in params")
 
-        # Identify numeric features
-        numeric_features_columns = [
-            field_dic["id"]
-            for field_dic in self.model.train_ds.fields
-            if field_dic["type"] == "Numeric"
-        ]
-        if self.params["features_columns"] is None:
-            features_list = numeric_features_columns
-        else:
-            features_list = self.params["features_columns"]
+        features_list = self.params["features_columns"]
+        if features_list is None:
+            features_list = self.model.train_ds.get_numeric_features_columns()
+
+        # Check if all elements from features_list are present in the numerical feature columns
+        all_present = all(
+            elem in self.model.train_ds.get_numeric_features_columns()
+            for elem in features_list
+        )
+        if not all_present:
+            raise ValueError(
+                "The list of feature columns provided do not match with training "
+                + "dataset numerical feature columns"
+            )
 
         # Remove target column if it exist in the list
-        if self.model.train_ds.target_column in features_list:
-            features_list.remove(self.model.train_ds.target_column)
+        features_list = [
+            col for col in features_list if col != self.model.train_ds.target_column
+        ]
 
         train_df = self.model.train_ds.x.copy()
         train_y_true = self.model.train_ds.y
@@ -870,8 +898,9 @@ class RobustnessDiagnosis(ThresholdTest):
         test_results = []
         test_figures = []
 
-        results_headers = ["Perturbation Size", "Dataset Type", "Records"]
-        results_headers.extend(self.default_metrics.keys())
+        results_headers = ["Perturbation Size", "Dataset Type", "Records"] + list(
+            self.default_metrics.keys()
+        )
         results = {k: [] for k in results_headers}
 
         # Iterate scaling factor for the standard deviation list
@@ -881,10 +910,10 @@ class RobustnessDiagnosis(ThresholdTest):
 
             # Add noise to numeric features columns provided by user
             for feature in features_list:
-                temp_train_df[feature] = self.add_noise_std_dev(
+                temp_train_df[feature] = self._add_noise_std_dev(
                     temp_train_df[feature].to_list(), x_std_dev
                 )
-                temp_test_df[feature] = self.add_noise_std_dev(
+                temp_test_df[feature] = self._add_noise_std_dev(
                     temp_test_df[feature].to_list(), x_std_dev
                 )
 
@@ -907,15 +936,31 @@ class RobustnessDiagnosis(ThresholdTest):
             )
         )
 
+        train_acc = df.loc[(df["Dataset Type"] == "Training"), "accuracy"].values[0]
+        test_acc = df.loc[(df["Dataset Type"] == "Test"), "accuracy"].values[0]
+
+        df["Passed"] = np.where(
+            (df["Dataset Type"] == "Training")
+            & (df["accuracy"] >= (train_acc - accuracy_threshold)),
+            True,
+            np.where(
+                (df["Dataset Type"] == "Test")
+                & (df["accuracy"] >= (test_acc - accuracy_threshold)),
+                True,
+                False,
+            ),
+        )
         test_results.append(
             TestResult(
                 test_name="accuracy",
-                column=features_list[0],
+                column=features_list,
                 passed=True,
-                values=df.to_dict(orient="list"),
+                values=df.to_dict(),
             )
         )
-        return self.cache_results(test_results, passed=True, figures=test_figures)
+        return self.cache_results(
+            test_results, passed=df["Passed"].all(), figures=test_figures
+        )
 
     def _compute_metrics(
         self,
@@ -946,7 +991,7 @@ class RobustnessDiagnosis(ThresholdTest):
         for metric, metric_fn in self.default_metrics.items():
             results[metric].append(metric_fn(y_true.values, y_prediction) * 100)
 
-    def add_noise_std_dev(
+    def _add_noise_std_dev(
         self, values: List[float], x_std_dev: float
     ) -> Tuple[List[float], float]:
         """

{validmind-1.11.4 → validmind-1.11.6}/validmind/vm_models/model.py

@@ -9,6 +9,7 @@ from .dataset import Dataset
 # import torch.nn as nn
 
 SUPPORTED_MODEL_TYPES = [
+    "catboost.CatBoostClassifier",
     "pytorch.PyTorchModel",
     "sklearn.LogisticRegression",
     "sklearn.LinearRegression",