workbench 0.8.184__tar.gz → 0.8.186__tar.gz

{workbench-0.8.184 → workbench-0.8.186}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: workbench
-Version: 0.8.184
+Version: 0.8.186
 Summary: Workbench: A Dashboard and Python API for creating and deploying AWS SageMaker Model Pipelines
 Author-email: SuperCowPowers LLC <support@supercowpowers.com>
 License-Expression: MIT

{workbench-0.8.184 → workbench-0.8.186}/aws_setup/workbench_compute/workbench_compute/workbench_compute_stack.py

@@ -9,6 +9,7 @@ from aws_cdk import (
     aws_sqs as sqs,
     aws_lambda as lambda_,
     aws_lambda_event_sources as lambda_events,
+    aws_logs as logs,
     Duration,
     Size,
 )
@@ -95,6 +96,7 @@ class WorkbenchComputeStack(Stack):
             compute_environment_name="workbench-compute-env",
             vpc=vpc,
             vpc_subnets=vpc_subnets,
+            maxv_cpus=16,  # Limit to 16 vCPU to minimize AWS Throttling issues
         )
 
     def create_batch_job_queue(self) -> batch.JobQueue:
@@ -192,6 +194,7 @@ class WorkbenchComputeStack(Stack):
             handler="index.lambda_handler",
             code=lambda_.Code.from_inline(self._get_lambda_code()),
             timeout=Duration.minutes(5),
+            log_retention=logs.RetentionDays.ONE_WEEK,
             environment={
                 "WORKBENCH_BUCKET": self.workbench_bucket,
                 "JOB_QUEUE": self.batch_job_queue.job_queue_name,
@@ -241,8 +244,6 @@ class WorkbenchComputeStack(Stack):
                         script_path = message['script_path']  # s3://bucket/path/to/script.py
                         size = message.get('size', 'small')
                         extra_env = message.get('environment', {})
-                        model_outputs = message.get('MODEL_OUTPUTS', [])
-                        endpoint_outputs = message.get('ENDPOINT_OUTPUTS', [])
 
                         script_name = Path(script_path).stem
                         job_name = f"workbench_{script_name}_{datetime.now():%Y%m%d_%H%M%S}"
@@ -257,14 +258,7 @@ class WorkbenchComputeStack(Stack):
                             *[{'name': k, 'value': v} for k, v in extra_env.items()]
                         ]
 
-                        # Add MODEL_OUTPUTS as comma-separated list if provided
-                        if model_outputs:
-                            env_vars.append({'name': 'MODEL_OUTPUTS', 'value': ','.join(model_outputs)})
-
-                        # Add ENDPOINT_OUTPUTS as comma-separated list if provided
-                        if endpoint_outputs:
-                            env_vars.append({'name': 'ENDPOINT_OUTPUTS', 'value': ','.join(endpoint_outputs)})
-
+                        # Submit Batch job to the queue
                         response = batch.submit_job(
                             jobName=job_name,
                             jobQueue=JOB_QUEUE,

workbench-0.8.186/scripts/delete_data_capture.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+import boto3
+import argparse
+from collections import defaultdict
+
+# Workbench Imports
+from workbench.utils.config_manager import ConfigManager
+
+
+def list_and_delete_data_capture(bucket, preview=True):
+    s3 = boto3.client("s3")
+    prefix = "endpoints/"
+
+    # Get all endpoint prefixes
+    paginator = s3.get_paginator("list_objects_v2")
+    endpoints = set()
+
+    for page in paginator.paginate(Bucket=bucket, Prefix=prefix, Delimiter="/"):
+        for common_prefix in page.get("CommonPrefixes", []):
+            endpoint_prefix = common_prefix["Prefix"]
+            if endpoint_prefix != prefix:
+                endpoints.add(endpoint_prefix)
+
+    # Count and optionally delete files per endpoint
+    counts = defaultdict(int)
+    to_delete = []
+
+    for endpoint_prefix in endpoints:
+        data_capture_prefix = f"{endpoint_prefix}data_capture/"
+
+        for page in paginator.paginate(Bucket=bucket, Prefix=data_capture_prefix):
+            for obj in page.get("Contents", []):
+                if obj["Key"].endswith(".jsonl"):
+                    endpoint_name = endpoint_prefix.split("/")[-2]
+                    counts[endpoint_name] += 1
+                    to_delete.append(obj["Key"])
+
+    # Display results
+    if counts:
+        print(f"\n{'Endpoint':<50} {'Files'}")
+        print("-" * 60)
+        for endpoint, count in sorted(counts.items()):
+            print(f"{endpoint:<50} {count:>6}")
+        print("-" * 60)
+        print(f"{'TOTAL':<50} {sum(counts.values()):>6}\n")
+    else:
+        print("No .jsonl files found in data_capture directories")
+        return
+
+    # Delete if not preview mode
+    if not preview:
+        print("Deleting files...")
+        # Delete in batches of 1000
+        for i in range(0, len(to_delete), 1000):
+            batch = to_delete[i : i + 1000]
+            s3.delete_objects(Bucket=bucket, Delete={"Objects": [{"Key": k} for k in batch]})
+        print(f"Deleted {len(to_delete)} files")
+    else:
+        print("PREVIEW MODE - no files deleted")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Remove data capture files from SageMaker endpoints")
+    parser.add_argument("--delete", action="store_true", help="Actually delete files (default is preview)")
+    args = parser.parse_args()
+
+    # Get the default S3 bucket from Workbench config
+    config = ConfigManager()
+    bucket = config.get_config("WORKBENCH_BUCKET")
+    print(f"Using S3 bucket: {bucket}")
+
+    list_and_delete_data_capture(bucket, preview=not args.delete)

workbench-0.8.186/scripts/sagemaker_feature_group_issue.py

@@ -0,0 +1,61 @@
+import pandas as pd
+import sagemaker
+from sagemaker.feature_store.feature_group import FeatureGroup
+import time
+import multiprocessing
+import multiprocess
+
+# Toggle this flag to test spawn mode fix
+USE_SPAWN_MODE = True  # Set to True to fix the Tahoe hang issue
+
+if __name__ == "__main__":
+    if USE_SPAWN_MODE:
+        print("Using SPAWN mode (fix for Tahoe)")
+        multiprocessing.set_start_method("spawn", force=True)
+        multiprocess.set_start_method("spawn", force=True)
+    else:
+        print("Using default fork mode (will hang on Tahoe)")
+
+    # Create fake data
+    data = pd.DataFrame(
+        {
+            "record_id": [f"id_{i}" for i in range(10)],
+            "feature_1": [float(i) for i in range(10)],
+            "feature_2": [float(i * 2) for i in range(10)],
+            "event_time": [time.time()] * 10,
+        }
+    )
+
+    # Setup SageMaker session
+    sagemaker_session = sagemaker.Session()
+
+    # Define feature group
+    feature_group_name = "temp_delete_me"
+    feature_group = FeatureGroup(name=feature_group_name, sagemaker_session=sagemaker_session)
+
+    # Create feature definitions
+    feature_group.load_feature_definitions(data_frame=data)
+
+    # Create feature group
+    print("Creating feature group...")
+    feature_group.create(
+        s3_uri=f"s3://{sagemaker_session.default_bucket()}/featurestore",
+        record_identifier_name="record_id",
+        event_time_feature_name="event_time",
+        role_arn=sagemaker.get_execution_role(),
+        enable_online_store=True,
+    )
+
+    # Wait for feature group to be created (can take 1-2 minutes)
+    print("Waiting for feature group to be ready...")
+    status = feature_group.describe().get("FeatureGroupStatus")
+    while status == "Creating":
+        print(f"Status: {status}... waiting 10 seconds")
+        time.sleep(10)
+        status = feature_group.describe().get("FeatureGroupStatus")
+    print(f"Feature group status: {status}")
+
+    # This will hang on macOS Tahoe with USE_SPAWN_MODE=False
+    print("Starting ingest...")
+    feature_group.ingest(data_frame=data, max_workers=2, max_processes=2, wait=True)
+    print("Ingest completed!")

workbench-0.8.186/scripts/test_feature_set_creation.py

@@ -0,0 +1,20 @@
+import awswrangler as wr
+
+# Workbench Imports
+from workbench.core.transforms.pandas_transforms import PandasToFeatures
+
+import multiprocessing
+import multiprocess
+
+if __name__ == "__main__":
+    multiprocessing.set_start_method("spawn", force=True)
+    multiprocess.set_start_method("spawn", force=True)
+
+    s3_path = "s3://workbench-public-data/comp_chem/aqsol_public_data.csv"
+    aqsol_data = wr.s3.read_csv(s3_path)
+    aqsol_data.columns = aqsol_data.columns.str.lower()
+
+    to_features = PandasToFeatures("temp_delete_me")
+    to_features.set_input(aqsol_data, id_column="id")
+    to_features.set_output_tags(["aqsol", "public"])
+    to_features.transform()

{workbench-0.8.184 → workbench-0.8.186}/src/workbench/core/artifacts/endpoint_core.py

@@ -475,17 +475,20 @@ class EndpointCore(Artifact):
             training_df = fs.view("training").pull_dataframe()
 
             # Run inference on the endpoint to get UQ outputs
-            full_inference_df = self.inference(training_df)
+            uq_df = self.inference(training_df)
 
             # Identify UQ-specific columns (quantiles and prediction_std)
-            uq_columns = [col for col in full_inference_df.columns if col.startswith("q_") or col == "prediction_std"]
+            uq_columns = [col for col in uq_df.columns if col.startswith("q_") or col == "prediction_std"]
 
             # Merge UQ columns with out-of-fold predictions
             if uq_columns:
-                # Keep id_column and UQ columns, drop 'prediction' to avoid conflict
-                merge_columns = [id_column] + uq_columns
-                uq_df = full_inference_df[merge_columns]
+                # Keep id_column and UQ columns, drop 'prediction' to avoid conflict when merging
+                uq_df = uq_df[[id_column] + uq_columns]
 
+                # Drop duplicates in uq_df based on id_column
+                uq_df = uq_df.drop_duplicates(subset=[id_column])
+
+                # Merge UQ columns into out_of_fold_df
                 out_of_fold_df = pd.merge(out_of_fold_df, uq_df, on=id_column, how="left")
                 additional_columns = uq_columns
                 self.log.info(f"Added UQ columns: {', '.join(additional_columns)}")
@@ -990,9 +993,9 @@ class EndpointCore(Artifact):
         self.upsert_workbench_meta({"workbench_input": input})
 
     def delete(self):
-        """ "Delete an existing Endpoint: Underlying Models, Configuration, and Endpoint"""
+        """Delete an existing Endpoint: Underlying Models, Configuration, and Endpoint"""
         if not self.exists():
-            self.log.warning(f"Trying to delete an Model that doesn't exist: {self.name}")
+            self.log.warning(f"Trying to delete an Endpoint that doesn't exist: {self.name}")
 
         # Remove this endpoint from the list of registered endpoints
         self.log.info(f"Removing {self.name} from the list of registered endpoints...")

{workbench-0.8.184 → workbench-0.8.186}/src/workbench/core/artifacts/model_core.py

@@ -899,7 +899,7 @@ class ModelCore(Artifact):
     def delete(self):
         """Delete the Model Packages and the Model Group"""
         if not self.exists():
-            self.log.warning(f"Trying to delete an Model that doesn't exist: {self.name}")
+            self.log.warning(f"Trying to delete a Model that doesn't exist: {self.name}")
 
         # Call the Class Method to delete the Model Group
         ModelCore.managed_delete(model_group_name=self.name)

{workbench-0.8.184 → workbench-0.8.186}/src/workbench/core/transforms/pandas_transforms/pandas_to_features.py

@@ -327,9 +327,36 @@ class PandasToFeatures(Transform):
         self.delete_existing()
         self.output_feature_group = self.create_feature_group()
 
+    def mac_spawn_hack(self):
+        """Workaround for macOS Tahoe fork/spawn issue with SageMaker FeatureStore ingest.
+
+        See: https://github.com/aws/sagemaker-python-sdk/issues/5312
+        macOS Tahoe 26+ has issues with forked processes creating boto3 sessions.
+        This forces spawn mode on macOS to avoid the hang.
+        """
+        import platform
+
+        if platform.system() == "Darwin":  # macOS
+            self.log.warning("macOS detected, forcing 'spawn' mode for multiprocessing (Tahoe hang workaround)")
+            import multiprocessing
+
+            try:
+                import multiprocess
+
+                multiprocess.set_start_method("spawn", force=True)
+            except (RuntimeError, ImportError):
+                pass  # Already set or multiprocess not available
+            try:
+                multiprocessing.set_start_method("spawn", force=True)
+            except RuntimeError:
+                pass  # Already set
+
     def transform_impl(self):
         """Transform Implementation: Ingest the data into the Feature Group"""
 
+        # Workaround for macOS Tahoe hang issue
+        self.mac_spawn_hack()
+
         # Now we actually push the data into the Feature Group (called ingestion)
         self.log.important(f"Ingesting rows into Feature Group {self.output_name}...")
         ingest_manager = self.output_feature_group.ingest(self.output_df, max_workers=8, max_processes=4, wait=False)

{workbench-0.8.184 → workbench-0.8.186}/src/workbench/model_scripts/uq_models/generated_model_script.py

@@ -466,7 +466,7 @@ def predict_fn(df, models) -> pd.DataFrame:
     df["q_50"] = df["prediction"]
 
     # Calculate a pseudo-standard deviation from the 68% interval width
-    df["prediction_std"] = (df["q_84"] - df["q_16"]) / 2.0
+    df["prediction_std"] = (df["q_84"] - df["q_16"]).abs() / 2.0
 
     # Reorder the quantile columns for easier reading
     quantile_cols = ["q_025", "q_05", "q_10", "q_16", "q_25", "q_75", "q_84", "q_90", "q_95", "q_975"]

{workbench-0.8.184 → workbench-0.8.186}/src/workbench/model_scripts/uq_models/mapie.template

@@ -466,7 +466,7 @@ def predict_fn(df, models) -> pd.DataFrame:
     df["q_50"] = df["prediction"]
 
     # Calculate a pseudo-standard deviation from the 68% interval width
-    df["prediction_std"] = (df["q_84"] - df["q_16"]) / 2.0
+    df["prediction_std"] = (df["q_84"] - df["q_16"]).abs() / 2.0
 
     # Reorder the quantile columns for easier reading
     quantile_cols = ["q_025", "q_05", "q_10", "q_16", "q_25", "q_75", "q_84", "q_90", "q_95", "q_975"]

{workbench-0.8.184 → workbench-0.8.186}/src/workbench/scripts/ml_pipeline_sqs.py

@@ -13,14 +13,21 @@ cm = ConfigManager()
 workbench_bucket = cm.get_config("WORKBENCH_BUCKET")
 
 
-def submit_to_sqs(script_path: str, size: str = "small", realtime: bool = False, recreate: bool = False) -> None:
+def submit_to_sqs(
+    script_path: str, size: str = "small", realtime: bool = False, dt: bool = False, promote: bool = False
+) -> None:
     """
     Upload script to S3 and submit message to SQS queue for processing.
+
     Args:
         script_path: Local path to the ML pipeline script
         size: Job size tier - "small" (default), "medium", or "large"
-        realtime: If True, sets serverless=False for real-time processing (default: False, meaning serverless=True)
-        recreate: If True, sets RECREATE=True in environment (default: False)
+        realtime: If True, sets serverless=False for real-time processing (default: False)
+        dt: If True, sets DT=True in environment (default: False)
+        promote: If True, sets PROMOTE=True in environment (default: False)
+
+    Raises:
+        ValueError: If size is invalid or script file not found
     """
     print(f"\n{'=' * 60}")
     print("🚀  SUBMITTING ML PIPELINE JOB")
@@ -36,7 +43,8 @@ def submit_to_sqs(script_path: str, size: str = "small", realtime: bool = False,
     print(f"📄  Script: {script_file.name}")
     print(f"📏  Size tier: {size}")
     print(f"⚡  Mode: {'Real-time' if realtime else 'Serverless'} (serverless={'False' if realtime else 'True'})")
-    print(f"🔄  Recreate: {recreate}")
+    print(f"🔄  DynamicTraining: {dt}")
+    print(f"🆕  Promote: {promote}")
     print(f"🪣  Bucket: {workbench_bucket}")
     sqs = AWSAccountClamp().boto3_session.client("sqs")
     script_name = script_file.name
@@ -94,14 +102,15 @@ def submit_to_sqs(script_path: str, size: str = "small", realtime: bool = False,
     message = {"script_path": s3_path, "size": size}
 
     # Set environment variables
-    message["environment"] = {"SERVERLESS": "False" if realtime else "True"}
-    if recreate:
-        message["environment"]["RECREATE"] = "True"
-
-    print("\n📨  Sending message to SQS...")
+    message["environment"] = {
+        "SERVERLESS": "False" if realtime else "True",
+        "DT": str(dt),
+        "PROMOTE": str(promote),
+    }
 
     # Send the message to SQS
     try:
+        print("\n📨  Sending message to SQS...")
         response = sqs.send_message(
             QueueUrl=queue_url,
             MessageBody=json.dumps(message, indent=2),
@@ -121,7 +130,8 @@ def submit_to_sqs(script_path: str, size: str = "small", realtime: bool = False,
     print(f"📄  Script: {script_name}")
     print(f"📏  Size: {size}")
     print(f"⚡  Mode: {'Real-time' if realtime else 'Serverless'} (SERVERLESS={'False' if realtime else 'True'})")
-    print(f"🔄  Recreate: {recreate}")
+    print(f"🔄  DynamicTraining: {dt}")
+    print(f"🆕  Promote: {promote}")
     print(f"🆔  Message ID: {message_id}")
     print("\n🔍  MONITORING LOCATIONS:")
     print(f"   • SQS Queue: AWS Console → SQS → {queue_name}")
@@ -144,13 +154,13 @@ def main():
         help="Create realtime endpoints (default is serverless)",
     )
     parser.add_argument(
-        "--recreate",
+        "--dt",
         action="store_true",
-        help="Set RECREATE=True (will force recreation of resources)",
+        help="Set DT=True (models and endpoints will have '-dt' suffix)",
     )
     args = parser.parse_args()
     try:
-        submit_to_sqs(args.script_file, args.size, realtime=args.realtime, recreate=args.recreate)
+        submit_to_sqs(args.script_file, args.size, realtime=args.realtime, dt=args.dt)
     except Exception as e:
         print(f"\n❌  ERROR: {e}")
         log.error(f"Error: {e}")

{workbench-0.8.184 → workbench-0.8.186}/src/workbench/utils/xgboost_model_utils.py

@@ -386,6 +386,106 @@ def cross_fold_inference(workbench_model: Any, nfolds: int = 5) -> Tuple[Dict[st
     return metrics_dict, predictions_df
 
 
+def leave_one_out_inference(workbench_model: Any) -> pd.DataFrame:
+    """
+    Performs leave-one-out cross-validation (parallelized).
+    For datasets > 1000 rows, first identifies top 100 worst predictions via 10-fold CV,
+    then performs true leave-one-out on those 100 samples.
+    Each model trains on ALL data except one sample.
+    """
+    from workbench.api import FeatureSet
+    from joblib import Parallel, delayed
+    from tqdm import tqdm
+
+    def train_and_predict_one(model_params, is_classifier, X, y, train_idx, val_idx):
+        """Train on train_idx, predict on val_idx."""
+        model = xgb.XGBClassifier(**model_params) if is_classifier else xgb.XGBRegressor(**model_params)
+        model.fit(X[train_idx], y[train_idx])
+        return model.predict(X[val_idx])[0]
+
+    # Load model and get params
+    model_artifact_uri = workbench_model.model_data_url()
+    loaded_model = xgboost_model_from_s3(model_artifact_uri)
+    if loaded_model is None:
+        log.error("No XGBoost model found in the artifact.")
+        return pd.DataFrame()
+
+    if isinstance(loaded_model, (xgb.XGBClassifier, xgb.XGBRegressor)):
+        is_classifier = isinstance(loaded_model, xgb.XGBClassifier)
+        model_params = loaded_model.get_params()
+    elif isinstance(loaded_model, xgb.Booster):
+        log.warning("Deprecated: Loaded model is a Booster, wrapping in sklearn model.")
+        is_classifier = workbench_model.model_type.value == "classifier"
+        model_params = {"enable_categorical": True}
+    else:
+        log.error(f"Unexpected model type: {type(loaded_model)}")
+        return pd.DataFrame()
+
+    # Load and prepare data
+    fs = FeatureSet(workbench_model.get_input())
+    df = fs.view("training").pull_dataframe()
+    id_col = fs.id_column
+    target_col = workbench_model.target()
+    feature_cols = workbench_model.features()
+
+    # Convert string features to categorical
+    for col in feature_cols:
+        if df[col].dtype in ["object", "string"]:
+            df[col] = df[col].astype("category")
+
+    # Determine which samples to run LOO on
+    if len(df) > 1000:
+        log.important(f"Dataset has {len(df)} rows. Running 10-fold CV to identify top 1000 worst predictions...")
+        _, predictions_df = cross_fold_inference(workbench_model, nfolds=10)
+        predictions_df["residual_abs"] = np.abs(predictions_df[target_col] - predictions_df["prediction"])
+        worst_samples = predictions_df.nlargest(1000, "residual_abs")
+        worst_ids = worst_samples[id_col].values
+        loo_indices = df[df[id_col].isin(worst_ids)].index.values
+        log.important(f"Running leave-one-out CV on 1000 worst samples. Each model trains on {len(df)-1} rows...")
+    else:
+        log.important(f"Running leave-one-out CV on all {len(df)} samples...")
+        loo_indices = df.index.values
+
+    # Prepare full dataset for training
+    X_full = df[feature_cols].values
+    y_full = df[target_col].values
+
+    # Encode target if classifier
+    label_encoder = LabelEncoder() if is_classifier else None
+    if label_encoder:
+        y_full = label_encoder.fit_transform(y_full)
+
+    # Generate LOO splits
+    splits = []
+    for loo_idx in loo_indices:
+        train_idx = np.delete(np.arange(len(X_full)), loo_idx)
+        val_idx = np.array([loo_idx])
+        splits.append((train_idx, val_idx))
+
+    # Parallel execution
+    predictions = Parallel(n_jobs=4)(
+        delayed(train_and_predict_one)(model_params, is_classifier, X_full, y_full, train_idx, val_idx)
+        for train_idx, val_idx in tqdm(splits, desc="LOO CV")
+    )
+
+    # Build results dataframe
+    predictions_array = np.array(predictions)
+    if label_encoder:
+        predictions_array = label_encoder.inverse_transform(predictions_array.astype(int))
+
+    predictions_df = pd.DataFrame(
+        {
+            id_col: df.loc[loo_indices, id_col].values,
+            target_col: df.loc[loo_indices, target_col].values,
+            "prediction": predictions_array,
+        }
+    )
+
+    predictions_df["residual_abs"] = np.abs(predictions_df[target_col] - predictions_df["prediction"])
+
+    return predictions_df
+
+
 if __name__ == "__main__":
     """Exercise the Model Utilities"""
     from workbench.api import Model, FeatureSet

{workbench-0.8.184 → workbench-0.8.186}/src/workbench.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: workbench
-Version: 0.8.184
+Version: 0.8.186
 Summary: Workbench: A Dashboard and Python API for creating and deploying AWS SageMaker Model Pipelines
 Author-email: SuperCowPowers LLC <support@supercowpowers.com>
 License-Expression: MIT

{workbench-0.8.184 → workbench-0.8.186}/src/workbench.egg-info/SOURCES.txt

@@ -377,6 +377,7 @@ scripts/active_logs.sh
 scripts/ag_table_row_selection.py
 scripts/compare_dataframes.py
 scripts/compare_model_packages.py
+scripts/delete_data_capture.py
 scripts/endpoint_timing.py
 scripts/find_smiles_issues.py
 scripts/get_tags.py
@@ -387,8 +388,10 @@ scripts/monitor_event_bridge.py
 scripts/onboard_endpoints.py
 scripts/onboard_models.py
 scripts/pipeline_model_endpoint.py
+scripts/sagemaker_feature_group_issue.py
 scripts/seaborn_to_plotly.py
 scripts/test_feature_resolution.py
+scripts/test_feature_set_creation.py
 scripts/test_no_workbench_config.py
 scripts/admin/athena_ddl_mixed_case.py
 scripts/admin/check_all_glue_tables.py