workbench 0.8.165__tar.gz → 0.8.166__tar.gz

{workbench-0.8.165 → workbench-0.8.166}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: workbench
-Version: 0.8.165
+Version: 0.8.166
 Summary: Workbench: A Dashboard and Python API for creating and deploying AWS SageMaker Model Pipelines
 Author-email: SuperCowPowers LLC <support@supercowpowers.com>
 License-Expression: MIT
@@ -30,7 +30,7 @@ Requires-Dist: joblib>=1.3.2
 Requires-Dist: requests>=2.26.0
 Requires-Dist: rdkit>=2024.9.5
 Requires-Dist: mordredcommunity>=2.0.6
-Requires-Dist: workbench-bridges>=0.1.8
+Requires-Dist: workbench-bridges>=0.1.10
 Provides-Extra: ui
 Requires-Dist: plotly>=6.0.0; extra == "ui"
 Requires-Dist: dash>3.0.0; extra == "ui"

{workbench-0.8.165 → workbench-0.8.166}/applications/aws_dashboard/Dockerfile

@@ -7,7 +7,7 @@ EXPOSE 8000
 ENV PYTHONHASHSEED=0
 
 # Define workbench version
-ARG WORKBENCH_VERSION=0.8.159
+ARG WORKBENCH_VERSION=0.8.165
 
 # Install system packages and gunicorn in one layer
 RUN apt-get update && apt-get install -y --no-install-recommends \

{workbench-0.8.165 → workbench-0.8.166}/applications/aws_dashboard/pages/data_sources/layout.py

@@ -58,6 +58,7 @@ def data_sources_layout(
                             data_source_details,
                             type="dot",
                             color="#33aa33",
+                            delay_show=300,
                         ),
                         width=4,
                         className="text-break workbench-container",
@@ -67,12 +68,12 @@ def data_sources_layout(
                     dbc.Col(
                         [
                             dbc.Row(
-                                dcc.Loading(violin_plot, type="graph"),
+                                dcc.Loading(violin_plot, type="graph", delay_show=500),
                                 className="workbench-container",
                                 style={"margin": "20px 0px 10px 20px"},
                             ),
                             dbc.Row(
-                                dcc.Loading(correlation_matrix, type="graph"),
+                                dcc.Loading(correlation_matrix, type="graph", delay_show=500),
                                 className="workbench-container",
                                 style={"margin": "20px 0px 10px 20px"},
                             ),

{workbench-0.8.165 → workbench-0.8.166}/applications/aws_dashboard/pages/endpoints/layout.py

@@ -47,7 +47,7 @@ def endpoints_layout(
                 [
                     # Column 1: Endpoint Details
                     dbc.Col(
-                        dcc.Loading(endpoint_details, type="dot", color="#33aa33"),
+                        dcc.Loading(endpoint_details, type="dot", color="#33aa33", delay_show=300),
                         width=4,
                         className="text-break workbench-container",
                         style={"margin": "20px 0px 0px 0px", "padding": "20px"},
@@ -56,7 +56,7 @@ def endpoints_layout(
                     dbc.Col(
                         [
                             dbc.Row(
-                                dcc.Loading(endpoint_metrics, type="graph"),
+                                dcc.Loading(endpoint_metrics, type="graph", delay_show=500),
                                 className="workbench-container",
                                 style={"margin": "20px 0px 10px 20px"},
                             ),

{workbench-0.8.165 → workbench-0.8.166}/applications/aws_dashboard/pages/feature_sets/layout.py

@@ -46,11 +46,7 @@ def feature_sets_layout(
                 [
                     # Column 1: Feature Set Details
                     dbc.Col(
-                        dcc.Loading(
-                            feature_set_details,
-                            type="dot",
-                            color="#33aa33",
-                        ),
+                        dcc.Loading(feature_set_details, type="dot", color="#33aa33", delay_show=300),
                         width=4,
                         className="text-break workbench-container",
                         style={"margin": "20px 0px 0px 0px", "padding": "20px"},
@@ -59,12 +55,12 @@ def feature_sets_layout(
                     dbc.Col(
                         [
                             dbc.Row(
-                                dcc.Loading(violin_plot, type="graph"),
+                                dcc.Loading(violin_plot, type="graph", delay_show=500),
                                 className="workbench-container",
                                 style={"margin": "20px 0px 10px 20px"},
                             ),
                             dbc.Row(
-                                dcc.Loading(correlation_matrix, type="graph"),
+                                dcc.Loading(correlation_matrix, type="graph", delay_show=500),
                                 className="workbench-container",
                                 style={"margin": "20px 0px 10px 20px"},
                             ),

{workbench-0.8.165 → workbench-0.8.166}/applications/aws_dashboard/pages/models/layout.py

@@ -37,11 +37,7 @@ def models_layout(
                 [
                     # Column 1: Model Details
                     dbc.Col(
-                        dcc.Loading(
-                            model_details,
-                            type="dot",
-                            color="#33aa33",
-                        ),
+                        dcc.Loading(model_details, type="dot", color="#33aa33", delay_show=300),
                         width=5,
                         className="text-break workbench-container",
                         style={"margin": "20px 0px 0px 0px", "padding": "20px"},
@@ -50,12 +46,12 @@ def models_layout(
                     dbc.Col(
                         [
                             dbc.Row(
-                                dcc.Loading(model_plot, type="graph", style={"height": "400px"}),
+                                dcc.Loading(model_plot, type="graph", delay_show=500, style={"height": "400px"}),
                                 className="workbench-container",
                                 style={"margin": "20px 0px 10px 20px"},
                             ),
                             dbc.Row(
-                                dcc.Loading(shap_plot, type="graph"),
+                                dcc.Loading(shap_plot, type="graph", delay_show=500),
                                 className="workbench-container",
                                 style={"margin": "20px 0px 10px 20px"},
                             ),

{workbench-0.8.165 → workbench-0.8.166}/aws_setup/aws_account_check.py

@@ -65,9 +65,9 @@ class AWSAccountCheck:
             self.log.info(str(feature_group))
         print("\n")
 
-        # Check that the Glue Database exist
-        self.log.info("*** AWS Glue Database Check ***")
-        for catalog_db in ["workbench", "sagemaker_featurestore"]:
+        # Check that the Glue Databases exist
+        self.log.info("*** AWS Glue Databases Check ***")
+        for catalog_db in ["workbench", "sagemaker_featurestore", "inference_store"]:
             self.ensure_aws_catalog_db(catalog_db)
         print("\n")
 

{workbench-0.8.165 → workbench-0.8.166}/aws_setup/workbench_core/workbench_core/workbench_core_stack.py

@@ -220,12 +220,22 @@ class WorkbenchCoreStack(Stack):
             ],
         )
 
-    def glue_pass_role(self) -> iam.PolicyStatement:
-        """Allows us to specify the Workbench-Glue role when creating a Glue Job"""
+    def glue_connections(self) -> iam.PolicyStatement:
+        """Create a policy statement for Glue connection access.
+
+        Returns:
+            iam.PolicyStatement: The policy statement for Glue connection interactions.
+        """
         return iam.PolicyStatement(
-            actions=["iam:PassRole"],
-            resources=[f"arn:aws:iam::{self.account}:role/Workbench-GlueRole"],
-            conditions={"StringEquals": {"iam:PassedToService": "glue.amazonaws.com"}},
+            actions=[
+                "glue:GetConnection",
+            ],
+            resources=[
+                # GetConnection requires access to the catalog resource since the
+                # Glue Catalog is used for metadata storage and connection retrieval
+                f"arn:aws:glue:{self.region}:{self.account}:catalog",
+                f"arn:aws:glue:{self.region}:{self.account}:connection/*",
+            ],
         )
 
     @staticmethod
@@ -260,6 +270,54 @@ class WorkbenchCoreStack(Stack):
             resources=read_statement.resources,
         )
 
+    def glue_pass_role(self) -> iam.PolicyStatement:
+        """Allows us to specify the Workbench-Glue role when creating a Glue Job"""
+        return iam.PolicyStatement(
+            actions=["iam:PassRole"],
+            resources=[f"arn:aws:iam::{self.account}:role/Workbench-GlueRole"],
+            conditions={"StringEquals": {"iam:PassedToService": "glue.amazonaws.com"}},
+        )
+
+    ####################
+    #  VPC/Networking  #
+    ####################
+    @staticmethod
+    def vpc_discovery() -> iam.PolicyStatement:
+        """Create a policy statement for VPC resource discovery.
+
+        Returns:
+            iam.PolicyStatement: The policy statement for VPC subnet and security group discovery operations.
+        """
+        return iam.PolicyStatement(
+            actions=[
+                "ec2:DescribeSubnets",
+                "ec2:DescribeSecurityGroups",
+                "ec2:DescribeVpcEndpoints",
+                "ec2:DescribeRouteTables",
+            ],
+            resources=[
+                "*",  # EC2 describe operations require wildcard resources
+            ],
+        )
+
+    def vpc_network_interface_management(self) -> iam.PolicyStatement:
+        """Create a policy statement for VPC network interface management.
+        Returns:
+            iam.PolicyStatement: The policy statement for ENI creation and discovery in VPC.
+        """
+        return iam.PolicyStatement(
+            actions=[
+                "ec2:CreateNetworkInterface",
+                "ec2:DescribeNetworkInterfaces",
+                "ec2:CreateTags",
+            ],
+            resources=[
+                f"arn:aws:ec2:{self.region}:{self.account}:network-interface/*",
+                f"arn:aws:ec2:{self.region}:{self.account}:subnet/*",
+                f"arn:aws:ec2:{self.region}:{self.account}:security-group/*",
+            ],
+        )
+
     ####################
     #      Athena      #
     ####################
@@ -1038,6 +1096,9 @@ class WorkbenchCoreStack(Stack):
 
         # Add a subset of policies for the Glue Role
         glue_role.add_to_policy(self.glue_job_logs())
+        glue_role.add_to_policy(self.glue_connections())
+        glue_role.add_to_policy(self.vpc_discovery())
+        glue_role.add_to_policy(self.vpc_network_interface_management())
         glue_role.add_to_policy(self.parameter_store_full())
         glue_role.add_managed_policy(self.datasource_policy)
         glue_role.add_managed_policy(self.featureset_policy)

{workbench-0.8.165 → workbench-0.8.166}/aws_setup/workbench_dashboard_full/app.py

@@ -12,7 +12,7 @@ print(f"Account: {aws_account}")
 print(f"Region: {aws_region}")
 
 # When you want a different docker image change this line
-dashboard_image = "public.ecr.aws/m6i5k1r2/workbench_dashboard:v0_8_159_amd64"
+dashboard_image = "public.ecr.aws/m6i5k1r2/workbench_dashboard:v0_8_165_amd64"
 
 # Workbench Configuration
 try:

{workbench-0.8.165 → workbench-0.8.166}/aws_setup/workbench_dashboard_lite/app.py

@@ -13,7 +13,7 @@ print(f"Account: {aws_account}")
 print(f"Region: {aws_region}")
 
 # When you want a different docker image change this line
-dashboard_image = "public.ecr.aws/m6i5k1r2/workbench_dashboard:v0_8_159_amd64"
+dashboard_image = "public.ecr.aws/m6i5k1r2/workbench_dashboard:v0_8_165_amd64"
 
 # Workbench Configuration
 try:

{workbench-0.8.165 → workbench-0.8.166}/pyproject.toml

@@ -40,7 +40,7 @@ dependencies = [
     "requests >= 2.26.0",
     "rdkit>=2024.9.5",
     "mordredcommunity>=2.0.6",
-    "workbench-bridges>=0.1.8",
+    "workbench-bridges>=0.1.10",
 ]
 
 [project.optional-dependencies]

{workbench-0.8.165 → workbench-0.8.166}/src/workbench/api/meta.py

@@ -113,13 +113,16 @@ class Meta(CloudMeta):
         """
         return super().models(details=details)
 
-    def endpoints(self) -> pd.DataFrame:
+    def endpoints(self, details: bool = False) -> pd.DataFrame:
         """Get a summary of the Endpoints deployed in the Cloud Platform
 
+        Args:
+            details (bool, optional): Include detailed information. Defaults to False.
+
         Returns:
             pd.DataFrame: A summary of the Endpoints in the Cloud Platform
         """
-        return super().endpoints()
+        return super().endpoints(details=details)
 
     def pipelines(self) -> pd.DataFrame:
         """Get a summary of the ML Pipelines deployed in the Cloud Platform

{workbench-0.8.165 → workbench-0.8.166}/src/workbench/core/artifacts/monitor_core.py

@@ -2,6 +2,8 @@
 
 import logging
 import json
+import re
+from datetime import datetime
 from typing import Union, Tuple
 import pandas as pd
 from sagemaker import Predictor
@@ -283,67 +285,66 @@ class MonitorCore:
             self.log.error(f"Error checking data capture percentage: {e}")
             return None
 
-    def get_captured_data(self, max_files=None, add_timestamp=True) -> Tuple[pd.DataFrame, pd.DataFrame]:
+    def get_captured_data(self, from_date=None, add_timestamp=True) -> Tuple[pd.DataFrame, pd.DataFrame]:
         """
         Read and process captured data from S3.
 
         Args:
-            max_files (int, optional): Maximum number of files to process.
+            from_date (str, optional): Only process files from this date onwards (YYYY-MM-DD format).
                                        Defaults to None to process all files.
             add_timestamp (bool, optional): Whether to add a timestamp column to the DataFrame.
 
         Returns:
             Tuple[pd.DataFrame, pd.DataFrame]: Processed input and output DataFrames.
         """
-        # List files in the specified S3 path
         files = wr.s3.list_objects(self.data_capture_path)
         if not files:
             self.log.warning(f"No data capture files found in {self.data_capture_path}.")
             return pd.DataFrame(), pd.DataFrame()
 
-        self.log.info(f"Found {len(files)} files in {self.data_capture_path}.")
-
-        # Sort files by timestamp (assuming the naming convention includes timestamp)
+        # Filter by date if specified
+        if from_date:
+            from_date_obj = datetime.strptime(from_date, "%Y-%m-%d").date()
+            files = [f for f in files if self._file_date_filter(f, from_date_obj)]
+            self.log.info(f"Processing {len(files)} files from {from_date} onwards.")
+        else:
+            self.log.info(f"Processing all {len(files)} files.")
         files.sort()
 
-        # Select files to process
-        if max_files is None:
-            files_to_process = files
-            self.log.info(f"Processing all {len(files)} files.")
-        else:
-            files_to_process = files[-max_files:] if files else []
-            self.log.info(f"Processing the {len(files_to_process)} most recent file(s).")
-
-        # Process each file
-        all_input_dfs = []
-        all_output_dfs = []
-        for file_path in files_to_process:
-            self.log.info(f"Processing {file_path}...")
+        # Process files
+        all_input_dfs, all_output_dfs = [], []
+        for file_path in files:
             try:
-                # Read the JSON lines file
                 df = wr.s3.read_json(path=file_path, lines=True)
                 if not df.empty:
                     input_df, output_df = process_data_capture(df)
-                    # Generate a timestamp column if requested
                     if add_timestamp:
-                        # Get file metadata to extract last modified time
-                        file_metadata = wr.s3.describe_objects(path=file_path)
-                        timestamp = file_metadata[file_path]["LastModified"]
+                        timestamp = wr.s3.describe_objects(path=file_path)[file_path]["LastModified"]
                         output_df["timestamp"] = timestamp
-
-                    # Append the processed DataFrames to the lists
                     all_input_dfs.append(input_df)
                     all_output_dfs.append(output_df)
             except Exception as e:
-                self.log.warning(f"Error processing file {file_path}: {e}")
+                self.log.warning(f"Error processing {file_path}: {e}")
 
-        # Combine all DataFrames
-        if not all_input_dfs or not all_output_dfs:
-            self.log.warning("No valid data was processed from the captured files.")
+        if not all_input_dfs:
+            self.log.warning("No valid data was processed.")
             return pd.DataFrame(), pd.DataFrame()
 
         return pd.concat(all_input_dfs, ignore_index=True), pd.concat(all_output_dfs, ignore_index=True)
 
+    def _file_date_filter(self, file_path, from_date_obj):
+        """Extract date from S3 path and compare with from_date."""
+        try:
+            # Match YYYY/MM/DD pattern in the path
+            date_match = re.search(r"/(\d{4})/(\d{2})/(\d{2})/", file_path)
+            if date_match:
+                year, month, day = date_match.groups()
+                file_date = datetime(int(year), int(month), int(day)).date()
+                return file_date >= from_date_obj
+            return False  # No date pattern found
+        except ValueError:
+            return False
+
     def baseline_exists(self) -> bool:
         """
         Check if baseline files exist in S3.

{workbench-0.8.165 → workbench-0.8.166}/src/workbench/core/cloud_platform/aws/aws_meta.py

@@ -308,7 +308,7 @@ class AWSMeta:
                     "Status": endpoint_details.get("EndpointStatus", "-"),
                     "Config": endpoint_details.get("EndpointConfigName", "-"),
                     "Variant": endpoint_details["config"]["variant"],
-                    "Capture": str(endpoint_details.get("DataCaptureConfig", {}).get("EnableCapture", "False")),
+                    "Capture": str(endpoint_details.get("DataCaptureConfig", {}).get("EnableCapture", "-")),
                     "Samp(%)": str(endpoint_details.get("DataCaptureConfig", {}).get("CurrentSamplingPercentage", "-")),
                     "Tags": aws_tags.get("workbench_tags", "-"),
                     "Monitored": endpoint_details["monitored"],

{workbench-0.8.165 → workbench-0.8.166}/src/workbench/core/cloud_platform/aws/aws_session.py

@@ -10,7 +10,7 @@ import logging
 
 # Workbench Imports
 from workbench.utils.config_manager import ConfigManager
-from workbench.utils.execution_environment import running_on_lambda, running_on_glue
+from workbench_bridges.utils.execution_environment import running_on_lambda, running_on_glue
 
 # Attempt to import IPython-related utilities
 try:

{workbench-0.8.165 → workbench-0.8.166}/src/workbench/repl/workbench_shell.py

@@ -72,7 +72,7 @@ if not ConfigManager().config_okay():
 
 # Set the log level to important
 log = logging.getLogger("workbench")
-log.setLevel(IMPORTANT_LEVEL_NUM)
+log.setLevel(logging.INFO)
 log.addFilter(
     lambda record: not (
         record.getMessage().startswith("Async: Metadata") or record.getMessage().startswith("Updated Metadata")

{workbench-0.8.165 → workbench-0.8.166}/src/workbench/utils/aws_utils.py

@@ -55,7 +55,8 @@ def aws_throttle(func=None, retry_intervals=None):
     if func is None:
         return lambda f: aws_throttle(f, retry_intervals=retry_intervals)
 
-    service_hold_time = 2  # Seconds to wait before calling AWS function
+    # This is currently commented out (we might want to use it later)
+    # service_hold_time = 2  # Seconds to wait before calling AWS function
     default_intervals = [2**i for i in range(1, 9)]  # Default exponential backoff: 2, 4, 8... 256 seconds
     intervals = retry_intervals or default_intervals
 
@@ -64,8 +65,8 @@ def aws_throttle(func=None, retry_intervals=None):
         for attempt, delay in enumerate(intervals, start=1):
             try:
                 # Add sleep before calling AWS func if running as a service
-                if cm.running_as_service:
-                    time.sleep(service_hold_time)
+                # if cm.running_as_service:
+                #    time.sleep(service_hold_time)
                 return func(*args, **kwargs)
             except ClientError as e:
                 if e.response["Error"]["Code"] == "ThrottlingException":

{workbench-0.8.165 → workbench-0.8.166}/src/workbench/utils/cloudwatch_handler.py

@@ -5,7 +5,7 @@ import getpass
 import time  # For managing send intervals
 
 # Workbench imports
-from workbench.utils.execution_environment import (
+from workbench_bridges.utils.execution_environment import (
     running_on_lambda,
     running_on_glue,
     running_on_ecs,

{workbench-0.8.165 → workbench-0.8.166}/src/workbench/utils/config_manager.py

@@ -9,7 +9,7 @@ from typing import Any, Dict
 
 # Workbench imports
 from workbench.utils.license_manager import LicenseManager
-from workbench.utils.execution_environment import running_as_service
+from workbench_bridges.utils.execution_environment import running_as_service
 
 # Python 3.9 compatibility
 from workbench.utils.resource_utils import get_resource_path

{workbench-0.8.165 → workbench-0.8.166}/src/workbench.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: workbench
-Version: 0.8.165
+Version: 0.8.166
 Summary: Workbench: A Dashboard and Python API for creating and deploying AWS SageMaker Model Pipelines
 Author-email: SuperCowPowers LLC <support@supercowpowers.com>
 License-Expression: MIT
@@ -30,7 +30,7 @@ Requires-Dist: joblib>=1.3.2
 Requires-Dist: requests>=2.26.0
 Requires-Dist: rdkit>=2024.9.5
 Requires-Dist: mordredcommunity>=2.0.6
-Requires-Dist: workbench-bridges>=0.1.8
+Requires-Dist: workbench-bridges>=0.1.10
 Provides-Extra: ui
 Requires-Dist: plotly>=6.0.0; extra == "ui"
 Requires-Dist: dash>3.0.0; extra == "ui"

{workbench-0.8.165 → workbench-0.8.166}/src/workbench.egg-info/SOURCES.txt

@@ -637,7 +637,6 @@ src/workbench/utils/df_to_endpoint.py
 src/workbench/utils/ecs_info.py
 src/workbench/utils/endpoint_metrics.py
 src/workbench/utils/endpoint_utils.py
-src/workbench/utils/execution_environment.py
 src/workbench/utils/extract_model_artifact.py
 src/workbench/utils/fast_inference.py
 src/workbench/utils/glue_utils.py

{workbench-0.8.165 → workbench-0.8.166}/src/workbench.egg-info/requires.txt

@@ -13,7 +13,7 @@ joblib>=1.3.2
 requests>=2.26.0
 rdkit>=2024.9.5
 mordredcommunity>=2.0.6
-workbench-bridges>=0.1.8
+workbench-bridges>=0.1.10
 
 [:sys_platform == "win32"]
 pyreadline3