workbench 0.8.180__tar.gz → 0.8.181__tar.gz

{workbench-0.8.180 → workbench-0.8.181}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: workbench
-Version: 0.8.180
+Version: 0.8.181
 Summary: Workbench: A Dashboard and Python API for creating and deploying AWS SageMaker Model Pipelines
 Author-email: SuperCowPowers LLC <support@supercowpowers.com>
 License-Expression: MIT

{workbench-0.8.180 → workbench-0.8.181}/src/workbench/core/artifacts/endpoint_core.py

@@ -429,8 +429,6 @@ class EndpointCore(Artifact):
                 # For UQ Models we also capture the uncertainty metrics
                 if model_type in [ModelType.UQ_REGRESSOR]:
                     metrics = uq_metrics(prediction_df, target_column)
-
-                    # Now put into the Parameter Store Model Inference Namespace
                     self.param_store.upsert(f"/workbench/models/{model.name}/inference/{capture_name}", metrics)
 
         # Return the prediction DataFrame
@@ -453,6 +451,55 @@ class EndpointCore(Artifact):
         cross_fold_metrics, out_of_fold_df = cross_fold_inference(model, nfolds=nfolds)
         if cross_fold_metrics:
             self.param_store.upsert(f"/workbench/models/{model.name}/inference/cross_fold", cross_fold_metrics)
+
+        # Capture the results
+        capture_name = "full_cross_fold"
+        description = capture_name.replace("_", " ").title()
+        target_column = model.target()
+        model_type = model.model_type
+
+        # Get the id_column from the model's FeatureSet
+        fs = FeatureSetCore(model.get_input())
+        id_column = fs.id_column
+
+        # Is this a UQ Model? If so, run full inference and merge the results
+        additional_columns = []
+        if model_type == ModelType.UQ_REGRESSOR:
+            self.log.important("UQ Regressor detected, running full inference to get uncertainty estimates...")
+
+            # Get the training view dataframe for inference
+            training_df = fs.view("training").pull_dataframe()
+
+            # Run inference on the endpoint to get UQ outputs
+            full_inference_df = self.inference(training_df)
+
+            # Identify UQ-specific columns (quantiles and prediction_std)
+            uq_columns = [col for col in full_inference_df.columns if col.startswith("q_") or col == "prediction_std"]
+
+            # Merge UQ columns with out-of-fold predictions
+            if uq_columns:
+                # Keep id_column and UQ columns, drop 'prediction' to avoid conflict
+                merge_columns = [id_column] + uq_columns
+                uq_df = full_inference_df[merge_columns]
+
+                out_of_fold_df = pd.merge(out_of_fold_df, uq_df, on=id_column, how="left")
+                additional_columns = uq_columns
+                self.log.info(f"Added UQ columns: {', '.join(additional_columns)}")
+
+                # Also compute UQ metrics
+                metrics = uq_metrics(out_of_fold_df, target_column)
+                self.param_store.upsert(f"/workbench/models/{model.name}/inference/{capture_name}", metrics)
+
+        self._capture_inference_results(
+            capture_name,
+            out_of_fold_df,
+            target_column,
+            model_type,
+            pd.DataFrame([cross_fold_metrics["summary_metrics"]]),
+            description,
+            features=additional_columns,
+            id_column=id_column,
+        )
         return cross_fold_metrics, out_of_fold_df
 
     def fast_inference(self, eval_df: pd.DataFrame, threads: int = 4) -> pd.DataFrame:
@@ -648,6 +695,10 @@ class EndpointCore(Artifact):
     @staticmethod
     def _hash_dataframe(df: pd.DataFrame, hash_length: int = 8):
         # Internal: Compute a data hash for the dataframe
+        if df.empty:
+            return "--hash--"
+
+        # Sort the dataframe by columns to ensure consistent ordering
         df = df.copy()
         df = df.sort_values(by=sorted(df.columns.tolist()))
         row_hashes = pd.util.hash_pandas_object(df, index=False)
@@ -702,8 +753,8 @@ class EndpointCore(Artifact):
         wr.s3.to_csv(metrics, f"{inference_capture_path}/inference_metrics.csv", index=False)
 
         # Grab the target column, prediction column, any _proba columns, and the ID column (if present)
-        prediction_col = "prediction" if "prediction" in pred_results_df.columns else "predictions"
-        output_columns = [target_column, prediction_col]
+        output_columns = [target_column]
+        output_columns += [col for col in pred_results_df.columns if "prediction" in col]
 
         # Add any _proba columns to the output columns
         output_columns += [col for col in pred_results_df.columns if col.endswith("_proba")]
@@ -1134,6 +1185,10 @@ if __name__ == "__main__":
     # Run predictions using the fast_inference method
     fast_results = my_endpoint.fast_inference(my_eval_df)
 
+    # Test the cross_fold_inference method
+    print("Running Cross-Fold Inference...")
+    metrics, all_results = my_endpoint.cross_fold_inference()
+
     # Run Inference and metrics for a Classification Endpoint
     class_endpoint = EndpointCore("wine-classification")
     auto_predictions = class_endpoint.auto_inference()
@@ -1142,6 +1197,10 @@ if __name__ == "__main__":
     target = "wine_class"
     print(class_endpoint.generate_confusion_matrix(target, auto_predictions))
 
+    # Test the cross_fold_inference method
+    print("Running Cross-Fold Inference...")
+    metrics, all_results = class_endpoint.cross_fold_inference()
+
     # Test the class method delete (commented out for now)
     # from workbench.api import Model
     # model = Model("abalone-regression")

{workbench-0.8.180 → workbench-0.8.181}/src/workbench/model_scripts/custom_models/uq_models/generated_model_script.py

@@ -21,8 +21,8 @@ TEMPLATE_PARAMS = {
     "target": "solubility",
     "features": ['molwt', 'mollogp', 'molmr', 'heavyatomcount', 'numhacceptors', 'numhdonors', 'numheteroatoms', 'numrotatablebonds', 'numvalenceelectrons', 'numaromaticrings', 'numsaturatedrings', 'numaliphaticrings', 'ringcount', 'tpsa', 'labuteasa', 'balabanj', 'bertzct'],
     "compressed_features": [],
-    "train_all_data": False,
-    "hyperparameters": {'n_estimators': 200, 'max_depth': 6, 'learning_rate': 0.05, 'subsample': 0.7, 'colsample_bytree': 0.3, 'colsample_bylevel': 0.5, 'min_child_weight': 5, 'gamma': 0.2, 'reg_alpha': 0.5, 'reg_lambda': 2.0, 'scale_pos_weight': 1},
+    "train_all_data": True,
+    "hyperparameters": {},
 }
 
 
@@ -303,7 +303,7 @@ if __name__ == "__main__":
         print(f"  {conf_level * 100:.0f}% CI: Mean width={np.mean(widths):.3f}, Std={np.std(widths):.3f}")
 
     # Save the trained XGBoost model
-    xgb_model.save_model(os.path.join(args.model_dir, "xgb_model.json"))
+    joblib.dump(xgb_model, os.path.join(args.model_dir, "xgb_model.joblib"))
 
     # Save all MAPIE models
     for model_name, model in mapie_models.items():
@@ -349,9 +349,8 @@ def model_fn(model_dir) -> dict:
         config = json.load(fp)
 
     # Load XGBoost regressor
-    xgb_path = os.path.join(model_dir, "xgb_model.json")
-    xgb_model = XGBRegressor(enable_categorical=True)
-    xgb_model.load_model(xgb_path)
+    xgb_path = os.path.join(model_dir, "xgb_model.joblib")
+    xgb_model = joblib.load(xgb_path)
 
     # Load all MAPIE models
     mapie_models = {}

{workbench-0.8.180 → workbench-0.8.181}/src/workbench/model_scripts/custom_models/uq_models/mapie.template

@@ -303,7 +303,7 @@ if __name__ == "__main__":
         print(f"  {conf_level * 100:.0f}% CI: Mean width={np.mean(widths):.3f}, Std={np.std(widths):.3f}")
 
     # Save the trained XGBoost model
-    xgb_model.save_model(os.path.join(args.model_dir, "xgb_model.json"))
+    joblib.dump(xgb_model, os.path.join(args.model_dir, "xgb_model.joblib"))
 
     # Save all MAPIE models
     for model_name, model in mapie_models.items():
@@ -349,9 +349,8 @@ def model_fn(model_dir) -> dict:
         config = json.load(fp)
 
     # Load XGBoost regressor
-    xgb_path = os.path.join(model_dir, "xgb_model.json")
-    xgb_model = XGBRegressor(enable_categorical=True)
-    xgb_model.load_model(xgb_path)
+    xgb_path = os.path.join(model_dir, "xgb_model.joblib")
+    xgb_model = joblib.load(xgb_path)
 
     # Load all MAPIE models
     mapie_models = {}

{workbench-0.8.180 → workbench-0.8.181}/src/workbench/model_scripts/xgb_model/xgb_model.template

@@ -145,7 +145,7 @@ def convert_categorical_types(df: pd.DataFrame, features: list, category_mapping
 def decompress_features(
     df: pd.DataFrame, features: List[str], compressed_features: List[str]
 ) -> Tuple[pd.DataFrame, List[str]]:
-    """Prepare features for the model
+    """Prepare features for the model by decompressing bitstring features
 
     Args:
         df (pd.DataFrame): The features DataFrame
@@ -170,7 +170,7 @@ def decompress_features(
         )
 
     # Decompress the specified compressed features
-    decompressed_features = features
+    decompressed_features = features.copy()
     for feature in compressed_features:
         if (feature not in df.columns) or (feature not in features):
             print(f"Feature '{feature}' not in the features list, skipping decompression.")

{workbench-0.8.180 → workbench-0.8.181}/src/workbench/utils/model_utils.py

@@ -3,6 +3,7 @@
 import logging
 import pandas as pd
 import numpy as np
+from scipy.stats import spearmanr
 import importlib.resources
 from pathlib import Path
 import os
@@ -222,8 +223,8 @@ def uq_metrics(df: pd.DataFrame, target_col: str) -> Dict[str, Any]:
         lower_95, upper_95 = df["q_025"], df["q_975"]
         lower_90, upper_90 = df["q_05"], df["q_95"]
         lower_80, upper_80 = df["q_10"], df["q_90"]
-        lower_68 = df.get("q_16", 0)
-        upper_68 = df.get("q_84", 0)
+        lower_68 = df.get("q_16", df["q_10"])  # fallback to 80% interval
+        upper_68 = df.get("q_84", df["q_90"])  # fallback to 80% interval
         lower_50, upper_50 = df["q_25"], df["q_75"]
     elif "prediction_std" in df.columns:
         lower_95 = df["prediction"] - 1.96 * df["prediction_std"]
@@ -240,18 +241,16 @@ def uq_metrics(df: pd.DataFrame, target_col: str) -> Dict[str, Any]:
         raise ValueError(
             "Either quantile columns (q_025, q_975, q_25, q_75) or 'prediction_std' column must be present."
         )
-    avg_std = df["prediction_std"].mean()
     median_std = df["prediction_std"].median()
     coverage_95 = np.mean((df[target_col] >= lower_95) & (df[target_col] <= upper_95))
     coverage_90 = np.mean((df[target_col] >= lower_90) & (df[target_col] <= upper_90))
     coverage_80 = np.mean((df[target_col] >= lower_80) & (df[target_col] <= upper_80))
     coverage_68 = np.mean((df[target_col] >= lower_68) & (df[target_col] <= upper_68))
-    coverage_50 = np.mean((df[target_col] >= lower_50) & (df[target_col] <= upper_50))
-    avg_width_95 = np.mean(upper_95 - lower_95)
-    avg_width_90 = np.mean(upper_90 - lower_90)
-    avg_width_80 = np.mean(upper_80 - lower_80)
-    avg_width_50 = np.mean(upper_50 - lower_50)
-    avg_width_68 = np.mean(upper_68 - lower_68)
+    median_width_95 = np.median(upper_95 - lower_95)
+    median_width_90 = np.median(upper_90 - lower_90)
+    median_width_80 = np.median(upper_80 - lower_80)
+    median_width_50 = np.median(upper_50 - lower_50)
+    median_width_68 = np.median(upper_68 - lower_68)
 
     # --- CRPS (measures calibration + sharpness) ---
     z = (df[target_col] - df["prediction"]) / df["prediction_std"]
@@ -267,47 +266,43 @@ def uq_metrics(df: pd.DataFrame, target_col: str) -> Dict[str, Any]:
     )
     mean_is_95 = np.mean(is_95)
 
-    # --- Adaptive Calibration (correlation between errors and uncertainty) ---
+    # --- Interval to Error Correlation ---
     abs_residuals = np.abs(df[target_col] - df["prediction"])
-    width_95 = upper_95 - lower_95
-    adaptive_calibration = np.corrcoef(abs_residuals, width_95)[0, 1]
+    width_68 = upper_68 - lower_68
+
+    # Spearman correlation for robustness
+    interval_to_error_corr = spearmanr(width_68, abs_residuals)[0]
 
     # Collect results
     results = {
-        "coverage_50": coverage_50,
         "coverage_68": coverage_68,
         "coverage_80": coverage_80,
         "coverage_90": coverage_90,
         "coverage_95": coverage_95,
         "median_std": median_std,
-        "avg_std": avg_std,
-        "avg_width_50": avg_width_50,
-        "avg_width_68": avg_width_68,
-        "avg_width_80": avg_width_80,
-        "avg_width_90": avg_width_90,
-        "avg_width_95": avg_width_95,
-        # "crps": mean_crps,
-        # "interval_score_95": mean_is_95,
-        # "adaptive_calibration": adaptive_calibration,
+        "median_width_50": median_width_50,
+        "median_width_68": median_width_68,
+        "median_width_80": median_width_80,
+        "median_width_90": median_width_90,
+        "median_width_95": median_width_95,
+        "interval_to_error_corr": interval_to_error_corr,
         "n_samples": len(df),
     }
 
     print("\n=== UQ Metrics ===")
-    print(f"Coverage @ 50%: {coverage_50:.3f} (target: 0.50)")
     print(f"Coverage @ 68%: {coverage_68:.3f} (target: 0.68)")
     print(f"Coverage @ 80%: {coverage_80:.3f} (target: 0.80)")
     print(f"Coverage @ 90%: {coverage_90:.3f} (target: 0.90)")
     print(f"Coverage @ 95%: {coverage_95:.3f} (target: 0.95)")
     print(f"Median Prediction StdDev: {median_std:.3f}")
-    print(f"Avg Prediction StdDev: {avg_std:.3f}")
-    print(f"Average 50% Width: {avg_width_50:.3f}")
-    print(f"Average 68% Width: {avg_width_68:.3f}")
-    print(f"Average 80% Width: {avg_width_80:.3f}")
-    print(f"Average 90% Width: {avg_width_90:.3f}")
-    print(f"Average 95% Width: {avg_width_95:.3f}")
+    print(f"Median 50% Width: {median_width_50:.3f}")
+    print(f"Median 68% Width: {median_width_68:.3f}")
+    print(f"Median 80% Width: {median_width_80:.3f}")
+    print(f"Median 90% Width: {median_width_90:.3f}")
+    print(f"Median 95% Width: {median_width_95:.3f}")
     print(f"CRPS: {mean_crps:.3f} (lower is better)")
     print(f"Interval Score 95%: {mean_is_95:.3f} (lower is better)")
-    print(f"Adaptive Calibration: {adaptive_calibration:.3f} (higher is better, target: >0.5)")
+    print(f"Interval/Error Corr: {interval_to_error_corr:.3f} (higher is better, target: >0.5)")
     print(f"Samples: {len(df)}")
     return results
 

{workbench-0.8.180 → workbench-0.8.181}/src/workbench.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: workbench
-Version: 0.8.180
+Version: 0.8.181
 Summary: Workbench: A Dashboard and Python API for creating and deploying AWS SageMaker Model Pipelines
 Author-email: SuperCowPowers LLC <support@supercowpowers.com>
 License-Expression: MIT

{workbench-0.8.180 → workbench-0.8.181}/src/workbench.egg-info/SOURCES.txt

@@ -710,6 +710,29 @@ src/workbench/web_interface/page_views/main_page.py
 src/workbench/web_interface/page_views/models_page_view.py
 src/workbench/web_interface/page_views/page_view.py
 src/workbench/web_interface/page_views/pipelines_page_view.py
+test_artifacts/100_percent_models.py
+test_artifacts/create_aqsol_artifacts.py
+test_artifacts/create_aqsol_rt_artifacts.py
+test_artifacts/create_aqsol_uq_artifacts.py
+test_artifacts/create_basic_test_artifacts.py
+test_artifacts/create_copy_artifacts.py
+test_artifacts/create_empty_model_group.py
+test_artifacts/create_endpoints_for_timing.py
+test_artifacts/create_graph_artifacts.py
+test_artifacts/create_knn_artifacts.py
+test_artifacts/create_meta_endpoints.py
+test_artifacts/create_pipeline_model.py
+test_artifacts/create_proximity_models.py
+test_artifacts/create_quant_regression_artifacts.py
+test_artifacts/create_realtime_endpoint.py
+test_artifacts/create_training_adjusted_artifacts.py
+test_artifacts/create_wine_artifacts.py
+test_artifacts/delete_stress_test_artifacts.py
+test_artifacts/delete_test_artifacts.py
+test_artifacts/hyper_testing.py
+test_artifacts/hyper_uq_testing.py
+test_artifacts/regression_ensemble.py
+test_artifacts/stress_test_aws.py
 tests/README.md
 tests/requirements-dev.txt
 tests/artifacts/data_source_tests.py
@@ -724,29 +747,6 @@ tests/aws_account/aws_account_clamp_tests.py
 tests/cached/cached_artifact_tests.py
 tests/chem_info/test_tautomerization.py
 tests/corner_cases/end_to_end.py
-tests/create_test_artifacts/100_percent_models.py
-tests/create_test_artifacts/create_aqsol_artifacts.py
-tests/create_test_artifacts/create_aqsol_rt_artifacts.py
-tests/create_test_artifacts/create_aqsol_uq_artifacts.py
-tests/create_test_artifacts/create_basic_test_artifacts.py
-tests/create_test_artifacts/create_copy_artifacts.py
-tests/create_test_artifacts/create_empty_model_group.py
-tests/create_test_artifacts/create_endpoints_for_timing.py
-tests/create_test_artifacts/create_graph_artifacts.py
-tests/create_test_artifacts/create_knn_artifacts.py
-tests/create_test_artifacts/create_meta_endpoints.py
-tests/create_test_artifacts/create_pipeline_model.py
-tests/create_test_artifacts/create_proximity_models.py
-tests/create_test_artifacts/create_quant_regression_artifacts.py
-tests/create_test_artifacts/create_realtime_endpoint.py
-tests/create_test_artifacts/create_training_adjusted_artifacts.py
-tests/create_test_artifacts/create_wine_artifacts.py
-tests/create_test_artifacts/delete_stress_test_artifacts.py
-tests/create_test_artifacts/delete_test_artifacts.py
-tests/create_test_artifacts/hyper_testing.py
-tests/create_test_artifacts/hyper_uq_testing.py
-tests/create_test_artifacts/regression_ensemble.py
-tests/create_test_artifacts/stress_test_aws.py
 tests/df_store/df_store_tests.py
 tests/lambda/decompress_layers.py
 tests/pandas_tests/test_drift.py

{workbench-0.8.180/tests/create_test_artifacts → workbench-0.8.181/test_artifacts}/create_aqsol_uq_artifacts.py

@@ -23,7 +23,7 @@ if __name__ == "__main__":
     s3_path = "s3://workbench-public-data/comp_chem/aqsol_public_data.csv"
 
     # Recreate Flag in case you want to recreate the artifacts
-    recreate = True
+    recreate = False
 
     # Check if the Model already exist
     if recreate or not Model("aqsol-uq").exists():
@@ -41,6 +41,7 @@ if __name__ == "__main__":
 
         # Run auto-inference on the Endpoint
         end.auto_inference(capture=True)
+        end.cross_fold_inference()
 
     # Check if the Model already exist
     if recreate or not Model("aqsol-uq-100").exists():
@@ -58,3 +59,4 @@ if __name__ == "__main__":
 
         # Run auto-inference on the Endpoint
         end.auto_inference(capture=True)
+        end.cross_fold_inference()