workbench 0.8.162__tar.gz → 0.8.164__tar.gz

{workbench-0.8.162 → workbench-0.8.164}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: workbench
-Version: 0.8.162
+Version: 0.8.164
 Summary: Workbench: A Dashboard and Python API for creating and deploying AWS SageMaker Model Pipelines
 Author-email: SuperCowPowers LLC <support@supercowpowers.com>
 License-Expression: MIT
@@ -22,7 +22,7 @@ Requires-Dist: pandas>=2.2.1
 Requires-Dist: awswrangler>=3.4.0
 Requires-Dist: sagemaker>=2.143
 Requires-Dist: cryptography>=44.0.2
-Requires-Dist: ipython>=9.0.0
+Requires-Dist: ipython>=8.37.0
 Requires-Dist: pyreadline3; sys_platform == "win32"
 Requires-Dist: scikit-learn>=1.5.2
 Requires-Dist: xgboost>=3.0.3
@@ -168,7 +168,6 @@ Using Workbench will minimize the time and manpower needed to incorporate AWS ML
 
 ```
 pip install workbench             # Installs Workbench with Core Dependencies
-pip install 'workbench[ml-tools]' # + Shap and NetworkX
 pip install 'workbench[ui]'       # + Plotly/Dash
 pip install 'workbench[dev]'      # + Pytest/flake8/black
 pip install 'workbench[all]'      # + All the things :)

{workbench-0.8.162 → workbench-0.8.164}/README.md

@@ -104,7 +104,6 @@ Using Workbench will minimize the time and manpower needed to incorporate AWS ML
 
 ```
 pip install workbench             # Installs Workbench with Core Dependencies
-pip install 'workbench[ml-tools]' # + Shap and NetworkX
 pip install 'workbench[ui]'       # + Plotly/Dash
 pip install 'workbench[dev]'      # + Pytest/flake8/black
 pip install 'workbench[all]'      # + All the things :)

{workbench-0.8.162 → workbench-0.8.164}/aws_setup/workbench_core/workbench_core/workbench_core_stack.py

@@ -202,6 +202,24 @@ class WorkbenchCoreStack(Stack):
     #####################
     #     Glue Jobs     #
     #####################
+    def glue_job_logs(self) -> iam.PolicyStatement:
+        """Create a policy statement for Glue job CloudWatch logs.
+
+        Returns:
+            iam.PolicyStatement: The policy statement for Glue job log interactions.
+        """
+        return iam.PolicyStatement(
+            actions=[
+                "logs:CreateLogGroup",
+                "logs:CreateLogStream",
+                "logs:PutLogEvents",
+            ],
+            resources=[
+                f"arn:aws:logs:{self.region}:{self.account}:log-group:/aws-glue/*",
+                f"arn:aws:logs:{self.region}:{self.account}:log-group:/aws-glue/*:*",
+            ],
+        )
+
     def glue_pass_role(self) -> iam.PolicyStatement:
         """Allows us to specify the Workbench-Glue role when creating a Glue Job"""
         return iam.PolicyStatement(
@@ -363,8 +381,7 @@ class WorkbenchCoreStack(Stack):
             resources=[f"arn:aws:sagemaker:{self.region}:{self.account}:training-job/*"],
         )
 
-    @staticmethod
-    def model_training_logs() -> iam.PolicyStatement:
+    def model_training_logs(self) -> iam.PolicyStatement:
         """Create a policy statement for log interactions when training SageMaker models.
         Returns:
             iam.PolicyStatement: The policy statement for log interactions when training SageMaker models.
@@ -378,7 +395,10 @@ class WorkbenchCoreStack(Stack):
                 "logs:CreateLogStream",
                 "logs:PutLogEvents",
             ],
-            resources=["arn:aws:logs:*:*:log-group:/aws/sagemaker/*", "arn:aws:logs:*:*:log-group:/aws/sagemaker/*:*"],
+            resources=[
+                f"arn:aws:logs:{self.region}:{self.account}:log-group:/aws/sagemaker/*",
+                f"arn:aws:logs:{self.region}:{self.account}:log-group:/aws/sagemaker/*:*",
+            ],
         )
 
     #####################
@@ -999,6 +1019,7 @@ class WorkbenchCoreStack(Stack):
         )
 
         # Add a subset of policies for the Glue Role
+        glue_role.add_to_policy(self.glue_job_logs())
         glue_role.add_to_policy(self.parameter_store_full())
         glue_role.add_managed_policy(self.datasource_policy)
         glue_role.add_managed_policy(self.featureset_policy)

{workbench-0.8.162 → workbench-0.8.164}/pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
     "awswrangler >= 3.4.0",
     "sagemaker >= 2.143",
     "cryptography >= 44.0.2",
-    "ipython >=9.0.0",
+    "ipython >=8.37.0",
     "pyreadline3; sys_platform == 'win32'",
     "scikit-learn >=1.5.2",
     "xgboost >=3.0.3",

{workbench-0.8.162 → workbench-0.8.164}/src/workbench/api/endpoint.py

@@ -70,6 +70,17 @@ class Endpoint(EndpointCore):
         """
         return super().fast_inference(eval_df, threads=threads)
 
+    def cross_fold_inference(self, nfolds: int = 5) -> dict:
+        """Run cross-fold inference (only works for XGBoost models)
+
+        Args:
+            nfolds (int): The number of folds to use for cross-validation (default: 5)
+
+        Returns:
+            dict: A dictionary with fold results
+        """
+        return super().cross_fold_inference(nfolds)
+
 
 if __name__ == "__main__":
     """Exercise the Endpoint Class"""

{workbench-0.8.162 → workbench-0.8.164}/src/workbench/core/artifacts/endpoint_core.py

@@ -378,16 +378,17 @@ class EndpointCore(Artifact):
             self.log.important("No model associated with this endpoint, running 'no frills' inference...")
             return self.fast_inference(eval_df)
 
+        # Grab the model features and target column
+        model = ModelCore(self.model_name)
+        features = model.features()
+        target_column = model.target()
+
         # Run predictions on the evaluation data
-        prediction_df = self._predict(eval_df, drop_error_rows)
+        prediction_df = self._predict(eval_df, features, drop_error_rows)
         if prediction_df.empty:
             self.log.warning("No predictions were made. Returning empty DataFrame.")
             return prediction_df
 
-        # Get the target column
-        model = ModelCore(self.model_name)
-        target_column = model.target()
-
         # Sanity Check that the target column is present
         if target_column and (target_column not in prediction_df.columns):
             self.log.important(f"Target Column {target_column} not found in prediction_df!")
@@ -419,12 +420,6 @@ class EndpointCore(Artifact):
                     capture_name, prediction_df, target_column, model_type, metrics, description, features, id_column
                 )
 
-                # Capture CrossFold Inference Results
-                cross_fold_metrics = cross_fold_inference(model)
-                if cross_fold_metrics:
-                    # Now put into the Parameter Store Model Inference Namespace
-                    self.param_store.upsert(f"/workbench/models/{model.name}/inference/cross_fold", cross_fold_metrics)
-
                 # For UQ Models we also capture the uncertainty metrics
                 if model_type in [ModelType.UQ_REGRESSOR]:
                     metrics = uq_metrics(prediction_df, target_column)
@@ -435,6 +430,25 @@ class EndpointCore(Artifact):
         # Return the prediction DataFrame
         return prediction_df
 
+    def cross_fold_inference(self, nfolds: int = 5) -> dict:
+        """Run cross-fold inference (only works for XGBoost models)
+
+        Args:
+            nfolds (int): Number of folds to use for cross-fold (default: 5)
+
+        Returns:
+            dict: Dictionary with the cross-fold inference results
+        """
+
+        # Grab our model
+        model = ModelCore(self.model_name)
+
+        # Compute CrossFold Metrics
+        cross_fold_metrics = cross_fold_inference(model, nfolds=nfolds)
+        if cross_fold_metrics:
+            self.param_store.upsert(f"/workbench/models/{model.name}/inference/cross_fold", cross_fold_metrics)
+        return cross_fold_metrics
+
     def fast_inference(self, eval_df: pd.DataFrame, threads: int = 4) -> pd.DataFrame:
         """Run inference on the Endpoint using the provided DataFrame
 
@@ -450,11 +464,12 @@ class EndpointCore(Artifact):
         """
         return fast_inference(self.name, eval_df, self.sm_session, threads=threads)
 
-    def _predict(self, eval_df: pd.DataFrame, drop_error_rows: bool = False) -> pd.DataFrame:
-        """Internal: Run prediction on the given observations in the given DataFrame
+    def _predict(self, eval_df: pd.DataFrame, features: list[str], drop_error_rows: bool = False) -> pd.DataFrame:
+        """Internal: Run prediction on observations in the given DataFrame
 
         Args:
             eval_df (pd.DataFrame): DataFrame to run predictions on (must have superset of features)
+            features (list[str]): List of feature column names needed for prediction
             drop_error_rows (bool): If True, drop rows that had endpoint errors/issues (default=False)
         Returns:
             pd.DataFrame: Return the DataFrame with additional columns, prediction and any _proba columns
@@ -465,19 +480,12 @@ class EndpointCore(Artifact):
             self.log.warning("Evaluation DataFrame has 0 rows. No predictions to run.")
             return pd.DataFrame(columns=eval_df.columns)  # Return empty DataFrame with same structure
 
-        # Sanity check: Does the Model have Features?
-        features = ModelCore(self.model_name).features()
-        if not features:
-            self.log.warning("Model does not have features defined, using all columns in the DataFrame")
-        else:
-            # Sanity check: Does the DataFrame have the required features?
-            df_columns_lower = set(col.lower() for col in eval_df.columns)
-            features_lower = set(feature.lower() for feature in features)
-
-            # Check if the features are a subset of the DataFrame columns (case-insensitive)
-            if not features_lower.issubset(df_columns_lower):
-                missing_features = features_lower - df_columns_lower
-                raise ValueError(f"DataFrame does not contain required features: {missing_features}")
+        # Sanity check: Does the DataFrame have the required features?
+        df_columns_lower = set(col.lower() for col in eval_df.columns)
+        features_lower = set(feature.lower() for feature in features)
+        if not features_lower.issubset(df_columns_lower):
+            missing_features = features_lower - df_columns_lower
+            raise ValueError(f"DataFrame does not contain required features: {missing_features}")
 
         # Create our Endpoint Predictor Class
         predictor = Predictor(
@@ -713,18 +721,10 @@ class EndpointCore(Artifact):
             # Note: Unlike other dataframes here, we want to write the index (labels) to the CSV
             wr.s3.to_csv(conf_mtx, f"{inference_capture_path}/inference_cm.csv", index=True)
 
-        # Generate SHAP values for our Prediction Dataframe
-        # generate_shap_values(self.endpoint_name, model_type.value, pred_results_df, inference_capture_path)
-
         # Now recompute the details for our Model
-        self.log.important(f"Recomputing Details for {self.model_name} to show latest Inference Results...")
+        self.log.important(f"Loading inference metrics for {self.model_name}...")
         model = ModelCore(self.model_name)
         model._load_inference_metrics(capture_name)
-        model.details()
-
-        # Recompute the details so that inference model metrics are updated
-        self.log.important(f"Recomputing Details for {self.name} to show latest Inference Results...")
-        self.details()
 
     def regression_metrics(self, target_column: str, prediction_df: pd.DataFrame) -> pd.DataFrame:
         """Compute the performance metrics for this Endpoint
@@ -876,9 +876,11 @@ class EndpointCore(Artifact):
 
     def generate_confusion_matrix(self, target_column: str, prediction_df: pd.DataFrame) -> pd.DataFrame:
         """Compute the confusion matrix for this Endpoint
+
         Args:
             target_column (str): Name of the target column
             prediction_df (pd.DataFrame): DataFrame with the prediction results
+
         Returns:
             pd.DataFrame: DataFrame with the confusion matrix
         """
@@ -887,25 +889,20 @@ class EndpointCore(Artifact):
         prediction_col = "prediction" if "prediction" in prediction_df.columns else "predictions"
         y_pred = prediction_df[prediction_col]
 
-        # Check if our model has class labels, if not we'll use the unique labels in the prediction
-        class_labels = ModelCore(self.model_name).class_labels()
-        if class_labels is None:
-            class_labels = sorted(list(set(y_true) | set(y_pred)))
+        # Get model class labels
+        model_class_labels = ModelCore(self.model_name).class_labels()
 
-        # Compute the confusion matrix (sklearn confusion_matrix)
-        conf_mtx = confusion_matrix(y_true, y_pred, labels=class_labels)
+        # Use model labels if available, otherwise infer from data
+        if model_class_labels:
+            self.log.important("Using model class labels for confusion matrix ordering...")
+            labels = model_class_labels
+        else:
+            labels = sorted(list(set(y_true) | set(y_pred)))
 
-        # Create a DataFrame
-        conf_mtx_df = pd.DataFrame(conf_mtx, index=class_labels, columns=class_labels)
+        # Compute confusion matrix and create DataFrame
+        conf_mtx = confusion_matrix(y_true, y_pred, labels=labels)
+        conf_mtx_df = pd.DataFrame(conf_mtx, index=labels, columns=labels)
         conf_mtx_df.index.name = "labels"
-
-        # Check if our model has class labels. If so make the index and columns ordered
-        model_class_labels = ModelCore(self.model_name).class_labels()
-        if model_class_labels:
-            self.log.important("Reordering the confusion matrix based on model class labels...")
-            conf_mtx_df.index = pd.Categorical(conf_mtx_df.index, categories=model_class_labels, ordered=True)
-            conf_mtx_df.columns = pd.Categorical(conf_mtx_df.columns, categories=model_class_labels, ordered=True)
-            conf_mtx_df = conf_mtx_df.sort_index().sort_index(axis=1)
         return conf_mtx_df
 
     def endpoint_config_name(self) -> str:
@@ -1086,13 +1083,20 @@ if __name__ == "__main__":
     df = fs.pull_dataframe()[:100]
     cap_df = df.copy()
     cap_df.columns = [col.upper() for col in cap_df.columns]
-    my_endpoint._predict(cap_df)
+    my_endpoint.inference(cap_df)
 
     # Boolean Type Test
     df["bool_column"] = [random.choice([True, False]) for _ in range(len(df))]
-    result_df = my_endpoint._predict(df)
+    result_df = my_endpoint.inference(df)
     assert result_df["bool_column"].dtype == bool
 
+    # Missing Feature Test
+    missing_df = df.drop(columns=["length"])
+    try:
+        my_endpoint.inference(missing_df)
+    except ValueError as e:
+        print(f"Expected error for missing feature: {e}")
+
     # Run Auto Inference on the Endpoint (uses the FeatureSet)
     print("Running Auto Inference...")
     my_endpoint.auto_inference()
@@ -1107,6 +1111,9 @@ if __name__ == "__main__":
     my_eval_df = fs_evaluation_data(my_endpoint)
     pred_results = my_endpoint.inference(my_eval_df, capture_name="holdout_xyz")
 
+    # Run predictions using the fast_inference method
+    fast_results = my_endpoint.fast_inference(my_eval_df)
+
     # Run Inference and metrics for a Classification Endpoint
     class_endpoint = EndpointCore("wine-classification")
     auto_predictions = class_endpoint.auto_inference()
@@ -1115,9 +1122,6 @@ if __name__ == "__main__":
     target = "wine_class"
     print(class_endpoint.generate_confusion_matrix(target, auto_predictions))
 
-    # Run predictions using the fast_inference method
-    fast_results = my_endpoint.fast_inference(my_eval_df)
-
     # Test the class method delete (commented out for now)
     # from workbench.api import Model
     # model = Model("abalone-regression")

{workbench-0.8.162 → workbench-0.8.164}/src/workbench/repl/workbench_shell.py

@@ -1,4 +1,6 @@
+import IPython
 from IPython import start_ipython
+from distutils.version import LooseVersion
 from IPython.terminal.prompts import Prompts
 from IPython.terminal.ipapp import load_default_config
 from pygments.token import Token
@@ -202,7 +204,10 @@ class WorkbenchShell:
 
         # Start IPython with the config and commands in the namespace
         try:
-            ipython_argv = ["--no-tip", "--theme", "linux"]
+            if LooseVersion(IPython.__version__) >= LooseVersion("9.0.0"):
+                ipython_argv = ["--no-tip", "--theme", "linux"]
+            else:
+                ipython_argv = []
             start_ipython(ipython_argv, user_ns=locs, config=config)
         finally:
             spinner = self.spinner_start("Goodbye to AWS:")

{workbench-0.8.162 → workbench-0.8.164}/src/workbench/utils/workbench_logging.py

@@ -181,9 +181,6 @@ def logging_setup(color_logs=True):
         log.debug("Debugging enabled via WORKBENCH_DEBUG environment variable.")
     else:
         log.setLevel(logging.INFO)
-        # Note: Not using the ThrottlingFilter for now
-        # throttle_filter = ThrottlingFilter(rate_seconds=5)
-        # handler.addFilter(throttle_filter)
 
     # Suppress specific logger
     logging.getLogger("sagemaker.config").setLevel(logging.WARNING)

{workbench-0.8.162 → workbench-0.8.164}/src/workbench.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: workbench
-Version: 0.8.162
+Version: 0.8.164
 Summary: Workbench: A Dashboard and Python API for creating and deploying AWS SageMaker Model Pipelines
 Author-email: SuperCowPowers LLC <support@supercowpowers.com>
 License-Expression: MIT
@@ -22,7 +22,7 @@ Requires-Dist: pandas>=2.2.1
 Requires-Dist: awswrangler>=3.4.0
 Requires-Dist: sagemaker>=2.143
 Requires-Dist: cryptography>=44.0.2
-Requires-Dist: ipython>=9.0.0
+Requires-Dist: ipython>=8.37.0
 Requires-Dist: pyreadline3; sys_platform == "win32"
 Requires-Dist: scikit-learn>=1.5.2
 Requires-Dist: xgboost>=3.0.3
@@ -168,7 +168,6 @@ Using Workbench will minimize the time and manpower needed to incorporate AWS ML
 
 ```
 pip install workbench             # Installs Workbench with Core Dependencies
-pip install 'workbench[ml-tools]' # + Shap and NetworkX
 pip install 'workbench[ui]'       # + Plotly/Dash
 pip install 'workbench[dev]'      # + Pytest/flake8/black
 pip install 'workbench[all]'      # + All the things :)

{workbench-0.8.162 → workbench-0.8.164}/src/workbench.egg-info/requires.txt

@@ -6,7 +6,7 @@ pandas>=2.2.1
 awswrangler>=3.4.0
 sagemaker>=2.143
 cryptography>=44.0.2
-ipython>=9.0.0
+ipython>=8.37.0
 scikit-learn>=1.5.2
 xgboost>=3.0.3
 joblib>=1.3.2

{workbench-0.8.162 → workbench-0.8.164}/tests/create_test_artifacts/create_wine_artifacts.py

@@ -44,6 +44,7 @@ if __name__ == "__main__":
             description="Wine Classification Model",
         )
         m.set_owner("test")
+        m.set_class_labels(["TypeA", "TypeB", "TypeC"])
 
     # Create the wine classification Endpoint
     if recreate or not Endpoint("wine-classification").exists():