workbench 0.8.180__py3-none-any.whl → 0.8.182__py3-none-any.whl

workbench/core/artifacts/endpoint_core.py

@@ -420,8 +420,12 @@ class EndpointCore(Artifact):
 
             # Capture the inference results and metrics
             if capture_name is not None:
+
+                # If we don't have an id_column, we'll pull it from the model's FeatureSet
+                if id_column is None:
+                    fs = FeatureSetCore(model.get_input())
+                    id_column = fs.id_column
                 description = capture_name.replace("_", " ").title()
-                features = model.features()
                 self._capture_inference_results(
                     capture_name, prediction_df, target_column, model_type, metrics, description, features, id_column
                 )
@@ -429,8 +433,6 @@ class EndpointCore(Artifact):
                 # For UQ Models we also capture the uncertainty metrics
                 if model_type in [ModelType.UQ_REGRESSOR]:
                     metrics = uq_metrics(prediction_df, target_column)
-
-                    # Now put into the Parameter Store Model Inference Namespace
                     self.param_store.upsert(f"/workbench/models/{model.name}/inference/{capture_name}", metrics)
 
         # Return the prediction DataFrame
@@ -453,6 +455,55 @@ class EndpointCore(Artifact):
         cross_fold_metrics, out_of_fold_df = cross_fold_inference(model, nfolds=nfolds)
         if cross_fold_metrics:
             self.param_store.upsert(f"/workbench/models/{model.name}/inference/cross_fold", cross_fold_metrics)
+
+        # Capture the results
+        capture_name = "full_cross_fold"
+        description = capture_name.replace("_", " ").title()
+        target_column = model.target()
+        model_type = model.model_type
+
+        # Get the id_column from the model's FeatureSet
+        fs = FeatureSetCore(model.get_input())
+        id_column = fs.id_column
+
+        # Is this a UQ Model? If so, run full inference and merge the results
+        additional_columns = []
+        if model_type == ModelType.UQ_REGRESSOR:
+            self.log.important("UQ Regressor detected, running full inference to get uncertainty estimates...")
+
+            # Get the training view dataframe for inference
+            training_df = fs.view("training").pull_dataframe()
+
+            # Run inference on the endpoint to get UQ outputs
+            full_inference_df = self.inference(training_df)
+
+            # Identify UQ-specific columns (quantiles and prediction_std)
+            uq_columns = [col for col in full_inference_df.columns if col.startswith("q_") or col == "prediction_std"]
+
+            # Merge UQ columns with out-of-fold predictions
+            if uq_columns:
+                # Keep id_column and UQ columns, drop 'prediction' to avoid conflict
+                merge_columns = [id_column] + uq_columns
+                uq_df = full_inference_df[merge_columns]
+
+                out_of_fold_df = pd.merge(out_of_fold_df, uq_df, on=id_column, how="left")
+                additional_columns = uq_columns
+                self.log.info(f"Added UQ columns: {', '.join(additional_columns)}")
+
+                # Also compute UQ metrics
+                metrics = uq_metrics(out_of_fold_df, target_column)
+                self.param_store.upsert(f"/workbench/models/{model.name}/inference/{capture_name}", metrics)
+
+        self._capture_inference_results(
+            capture_name,
+            out_of_fold_df,
+            target_column,
+            model_type,
+            pd.DataFrame([cross_fold_metrics["summary_metrics"]]),
+            description,
+            features=additional_columns,
+            id_column=id_column,
+        )
         return cross_fold_metrics, out_of_fold_df
 
     def fast_inference(self, eval_df: pd.DataFrame, threads: int = 4) -> pd.DataFrame:
@@ -648,6 +699,10 @@ class EndpointCore(Artifact):
     @staticmethod
     def _hash_dataframe(df: pd.DataFrame, hash_length: int = 8):
         # Internal: Compute a data hash for the dataframe
+        if df.empty:
+            return "--hash--"
+
+        # Sort the dataframe by columns to ensure consistent ordering
         df = df.copy()
         df = df.sort_values(by=sorted(df.columns.tolist()))
         row_hashes = pd.util.hash_pandas_object(df, index=False)
@@ -702,8 +757,8 @@ class EndpointCore(Artifact):
         wr.s3.to_csv(metrics, f"{inference_capture_path}/inference_metrics.csv", index=False)
 
         # Grab the target column, prediction column, any _proba columns, and the ID column (if present)
-        prediction_col = "prediction" if "prediction" in pred_results_df.columns else "predictions"
-        output_columns = [target_column, prediction_col]
+        output_columns = [target_column]
+        output_columns += [col for col in pred_results_df.columns if "prediction" in col]
 
         # Add any _proba columns to the output columns
         output_columns += [col for col in pred_results_df.columns if col.endswith("_proba")]
@@ -713,7 +768,7 @@ class EndpointCore(Artifact):
 
         # Add the ID column
         if id_column and id_column in pred_results_df.columns:
-            output_columns.append(id_column)
+            output_columns.insert(0, id_column)
 
         # Write the predictions to our S3 Model Inference Folder
         self.log.info(f"Writing predictions to {inference_capture_path}/inference_predictions.csv")
@@ -1134,6 +1189,10 @@ if __name__ == "__main__":
     # Run predictions using the fast_inference method
     fast_results = my_endpoint.fast_inference(my_eval_df)
 
+    # Test the cross_fold_inference method
+    print("Running Cross-Fold Inference...")
+    metrics, all_results = my_endpoint.cross_fold_inference()
+
     # Run Inference and metrics for a Classification Endpoint
     class_endpoint = EndpointCore("wine-classification")
     auto_predictions = class_endpoint.auto_inference()
@@ -1142,6 +1201,10 @@ if __name__ == "__main__":
     target = "wine_class"
     print(class_endpoint.generate_confusion_matrix(target, auto_predictions))
 
+    # Test the cross_fold_inference method
+    print("Running Cross-Fold Inference...")
+    metrics, all_results = class_endpoint.cross_fold_inference()
+
     # Test the class method delete (commented out for now)
     # from workbench.api import Model
     # model = Model("abalone-regression")

workbench/model_scripts/custom_models/uq_models/generated_model_script.py

@@ -18,11 +18,11 @@ from typing import List, Tuple
 
 # Template Placeholders
 TEMPLATE_PARAMS = {
-    "target": "solubility",
-    "features": ['molwt', 'mollogp', 'molmr', 'heavyatomcount', 'numhacceptors', 'numhdonors', 'numheteroatoms', 'numrotatablebonds', 'numvalenceelectrons', 'numaromaticrings', 'numsaturatedrings', 'numaliphaticrings', 'ringcount', 'tpsa', 'labuteasa', 'balabanj', 'bertzct'],
+    "target": "udm_asy_res_free_percent",
+    "features": ['vsa_estate6', 'naromatom', 'mollogp', 'fr_nh2', 'mp', 'c2sp2', 'xch_3d', 'axp_6d', 'bcut2d_mrhi', 'fr_benzene', 'mz', 'slogp_vsa6', 'fr_halogen', 'bcut2d_mwhi', 'vsa_estate4', 'slogp_vsa3', 'estate_vsa5', 'minestateindex', 'axp_3dv', 'estate_vsa3', 'vsa_estate9', 'molwt', 'hallkieralpha', 'fpdensitymorgan1', 'peoe_vsa13', 'xpc_5d', 'bcut2d_chghi', 'peoe_vsa8', 'axp_0dv', 'axp_2d', 'chi2v', 'bcut2d_logphi', 'axp_5d', 'peoe_vsa2', 'estate_vsa6', 'qed', 'numrotatablebonds', 'xc_3dv', 'peoe_vsa3', 'balabanj', 'slogp_vsa5', 'mv', 'vsa_estate2', 'bcut2d_mwlow', 'xch_7d', 'chi3n', 'vsa_estate8', 'estate_vsa4', 'xp_7dv', 'fr_nh1', 'vsa_estate3', 'fr_ketone_topliss', 'minpartialcharge', 'phi', 'peoe_vsa10', 'vsa_estate7', 'estate_vsa7', 'tpsa', 'kappa3', 'kappa2', 'bcut2d_logplow', 'xch_6d', 'maxpartialcharge', 'vsa_estate1', 'peoe_vsa9', 'axp_1d', 'fr_ar_n', 'chi2n', 'vsa_estate5', 'xp_4dv', 'slogp_vsa10', 'num_stereobonds', 'peoe_vsa11', 'bcut2d_chglo', 'chi1v', 'peoe_vsa7', 'bertzct', 'axp_2dv', 'estate_vsa2', 'smr_vsa9', 'peoe_vsa6', 'num_s_centers', 'num_r_centers', 'xch_7dv', 'xc_5d', 'axp_4dv', 'xc_5dv', 'mi', 'xc_3d', 'fpdensitymorgan2', 'xp_0dv', 'nhohcount', 'numatomstereocenters', 'mse', 'smr_vsa3', 'peoe_vsa12', 'nocount', 'fpdensitymorgan3', 'minabsestateindex', 'bcut2d_mrlow', 'axp_5dv', 'sz', 'vsa_estate10', 'axp_3d', 'xch_6dv', 'xch_4d', 'xc_6d', 'estate_vsa8', 'mpe', 'smr_vsa7', 'numhdonors', 'smr_vsa1', 'xp_5d', 'fr_para_hydroxylation', 'chi3v', 'xpc_6dv', 'nbase', 'heavyatommolwt', 'avgipc', 'maxestateindex', 'smr_vsa6', 'fr_bicyclic', 'xc_4dv', 'xp_7d', 'smr_vsa5', 'xpc_4d', 'smr_vsa4', 'peoe_vsa4', 'numheteroatoms', 'fr_nhpyrrole', 'axp_4d', 'smr_vsa10', 'xp_6d', 'sps', 'mare', 'slogp_vsa2', 'axp_0d', 'slogp_vsa4', 'fr_al_oh', 'numheterocycles', 'labuteasa', 'xp_3d', 'chi4n', 'fractioncsp3', 'maxabspartialcharge', 'fr_al_oh_notert', 'peoe_vsa1', 'axp_7dv', 'slogp_vsa11', 'peoe_vsa5', 'xpc_5dv', 'xpc_6d', 'xp_2d', 'xp_3dv', 'fr_ndealkylation1', 'axp_7d', 'estate_vsa9', 'molmr', 'num_stereocenters', 'si', 'estate_vsa1', 'xc_6dv', 'chi0v', 'fr_oxazole', 'axp_6dv', 'xp_6dv', 'xp_4d', 'numaliphaticheterocycles', 'fr_imine', 'fr_imidazole', 'xp_5dv', 'fr_piperdine', 'slogp_vsa7', 'chi1', 'c1sp2', 'numaromaticheterocycles', 'xpc_4dv', 'c3sp2', 'fr_aniline', 'fr_piperzine', 'axp_1dv', 'xch_4dv', 'chi4v', 'chi1n', 'minabspartialcharge', 'slogp_vsa1', 'fr_nh0', 'chi0n', 'c2sp3', 'xc_4d', 'xch_5dv', 'peoe_vsa14', 'xch_5d', 'numsaturatedrings', 'fr_pyridine', 'kappa1', 'slogp_vsa8', 'xp_2dv', 'fr_ar_coo', 'numvalenceelectrons'],
     "compressed_features": [],
-    "train_all_data": False,
-    "hyperparameters": {'n_estimators': 200, 'max_depth': 6, 'learning_rate': 0.05, 'subsample': 0.7, 'colsample_bytree': 0.3, 'colsample_bylevel': 0.5, 'min_child_weight': 5, 'gamma': 0.2, 'reg_alpha': 0.5, 'reg_lambda': 2.0, 'scale_pos_weight': 1},
+    "train_all_data": True,
+    "hyperparameters": {},
 }
 
 
@@ -303,7 +303,7 @@ if __name__ == "__main__":
         print(f"  {conf_level * 100:.0f}% CI: Mean width={np.mean(widths):.3f}, Std={np.std(widths):.3f}")
 
     # Save the trained XGBoost model
-    xgb_model.save_model(os.path.join(args.model_dir, "xgb_model.json"))
+    joblib.dump(xgb_model, os.path.join(args.model_dir, "xgb_model.joblib"))
 
     # Save all MAPIE models
     for model_name, model in mapie_models.items():
@@ -349,9 +349,8 @@ def model_fn(model_dir) -> dict:
         config = json.load(fp)
 
     # Load XGBoost regressor
-    xgb_path = os.path.join(model_dir, "xgb_model.json")
-    xgb_model = XGBRegressor(enable_categorical=True)
-    xgb_model.load_model(xgb_path)
+    xgb_path = os.path.join(model_dir, "xgb_model.joblib")
+    xgb_model = joblib.load(xgb_path)
 
     # Load all MAPIE models
     mapie_models = {}

workbench/model_scripts/script_generation.py

@@ -117,8 +117,8 @@ def generate_model_script(template_params: dict) -> str:
         template_name = "xgb_model.template"
         model_script_dir = "xgb_model"
     elif template_params["model_type"] == ModelType.UQ_REGRESSOR:
-        template_name = "quant_regression.template"
-        model_script_dir = "quant_regression"
+        template_name = "mapie.template"
+        model_script_dir = "uq_models"
     elif template_params["model_type"] == ModelType.ENSEMBLE_REGRESSOR:
         template_name = "ensemble_xgb.template"
         model_script_dir = "ensemble_xgb"

workbench/model_scripts/uq_models/generated_model_script.py

@@ -0,0 +1,492 @@
+# Model: XGBoost for point predictions + LightGBM with MAPIE for conformalized intervals
+from mapie.regression import ConformalizedQuantileRegressor
+from lightgbm import LGBMRegressor
+from xgboost import XGBRegressor
+from sklearn.model_selection import train_test_split
+
+# Model Performance Scores
+from sklearn.metrics import mean_absolute_error, r2_score, root_mean_squared_error
+
+from io import StringIO
+import json
+import argparse
+import joblib
+import os
+import numpy as np
+import pandas as pd
+from typing import List, Tuple
+
+# Template Placeholders
+TEMPLATE_PARAMS = {
+    "target": "udm_asy_res_free_percent",
+    "features": ['vsa_estate6', 'naromatom', 'mollogp', 'fr_nh2', 'mp', 'c2sp2', 'xch_3d', 'axp_6d', 'bcut2d_mrhi', 'fr_benzene', 'mz', 'slogp_vsa6', 'fr_halogen', 'bcut2d_mwhi', 'vsa_estate4', 'slogp_vsa3', 'estate_vsa5', 'minestateindex', 'axp_3dv', 'estate_vsa3', 'vsa_estate9', 'molwt', 'hallkieralpha', 'fpdensitymorgan1', 'peoe_vsa13', 'xpc_5d', 'bcut2d_chghi', 'peoe_vsa8', 'axp_0dv', 'axp_2d', 'chi2v', 'bcut2d_logphi', 'axp_5d', 'peoe_vsa2', 'estate_vsa6', 'qed', 'numrotatablebonds', 'xc_3dv', 'peoe_vsa3', 'balabanj', 'slogp_vsa5', 'mv', 'vsa_estate2', 'bcut2d_mwlow', 'xch_7d', 'chi3n', 'vsa_estate8', 'estate_vsa4', 'xp_7dv', 'fr_nh1', 'vsa_estate3', 'fr_ketone_topliss', 'minpartialcharge', 'phi', 'peoe_vsa10', 'vsa_estate7', 'estate_vsa7', 'tpsa', 'kappa3', 'kappa2', 'bcut2d_logplow', 'xch_6d', 'maxpartialcharge', 'vsa_estate1', 'peoe_vsa9', 'axp_1d', 'fr_ar_n', 'chi2n', 'vsa_estate5', 'xp_4dv', 'slogp_vsa10', 'num_stereobonds', 'peoe_vsa11', 'bcut2d_chglo', 'chi1v', 'peoe_vsa7', 'bertzct', 'axp_2dv', 'estate_vsa2', 'smr_vsa9', 'peoe_vsa6', 'num_s_centers', 'num_r_centers', 'xch_7dv', 'xc_5d', 'axp_4dv', 'xc_5dv', 'mi', 'xc_3d', 'fpdensitymorgan2', 'xp_0dv', 'nhohcount', 'numatomstereocenters', 'mse', 'smr_vsa3', 'peoe_vsa12', 'nocount', 'fpdensitymorgan3', 'minabsestateindex', 'bcut2d_mrlow', 'axp_5dv', 'sz', 'vsa_estate10', 'axp_3d', 'xch_6dv', 'xch_4d', 'xc_6d', 'estate_vsa8', 'mpe', 'smr_vsa7', 'numhdonors', 'smr_vsa1', 'xp_5d', 'fr_para_hydroxylation', 'chi3v', 'xpc_6dv', 'nbase', 'heavyatommolwt', 'avgipc', 'maxestateindex', 'smr_vsa6', 'fr_bicyclic', 'xc_4dv', 'xp_7d', 'smr_vsa5', 'xpc_4d', 'smr_vsa4', 'peoe_vsa4', 'numheteroatoms', 'fr_nhpyrrole', 'axp_4d', 'smr_vsa10', 'xp_6d', 'sps', 'mare', 'slogp_vsa2', 'axp_0d', 'slogp_vsa4', 'fr_al_oh', 'numheterocycles', 'labuteasa', 'xp_3d', 'chi4n', 'fractioncsp3', 'maxabspartialcharge', 'fr_al_oh_notert', 'peoe_vsa1', 'axp_7dv', 'slogp_vsa11', 'peoe_vsa5', 'xpc_5dv', 'xpc_6d', 'xp_2d', 'xp_3dv', 'fr_ndealkylation1', 'axp_7d', 'estate_vsa9', 'molmr', 'num_stereocenters', 'si', 'estate_vsa1', 'xc_6dv', 'chi0v', 'fr_oxazole', 'axp_6dv', 'xp_6dv', 'xp_4d', 'numaliphaticheterocycles', 'fr_imine', 'fr_imidazole', 'xp_5dv', 'fr_piperdine', 'slogp_vsa7', 'chi1', 'c1sp2', 'numaromaticheterocycles', 'xpc_4dv', 'c3sp2', 'fr_aniline', 'fr_piperzine', 'axp_1dv', 'xch_4dv', 'chi4v', 'chi1n', 'minabspartialcharge', 'slogp_vsa1', 'fr_nh0', 'chi0n', 'c2sp3', 'xc_4d', 'xch_5dv', 'peoe_vsa14', 'xch_5d', 'numsaturatedrings', 'fr_pyridine', 'kappa1', 'slogp_vsa8', 'xp_2dv', 'fr_ar_coo', 'numvalenceelectrons'],
+    "compressed_features": [],
+    "train_all_data": True,
+    "hyperparameters": {},
+}
+
+
+# Function to check if dataframe is empty
+def check_dataframe(df: pd.DataFrame, df_name: str) -> None:
+    """
+    Check if the provided dataframe is empty and raise an exception if it is.
+
+    Args:
+        df (pd.DataFrame): DataFrame to check
+        df_name (str): Name of the DataFrame
+    """
+    if df.empty:
+        msg = f"*** The training data {df_name} has 0 rows! ***STOPPING***"
+        print(msg)
+        raise ValueError(msg)
+
+
+def match_features_case_insensitive(df: pd.DataFrame, model_features: list) -> pd.DataFrame:
+    """
+    Matches and renames DataFrame columns to match model feature names (case-insensitive).
+    Prioritizes exact matches, then case-insensitive matches.
+
+    Raises ValueError if any model features cannot be matched.
+    """
+    df_columns_lower = {col.lower(): col for col in df.columns}
+    rename_dict = {}
+    missing = []
+    for feature in model_features:
+        if feature in df.columns:
+            continue  # Exact match
+        elif feature.lower() in df_columns_lower:
+            rename_dict[df_columns_lower[feature.lower()]] = feature
+        else:
+            missing.append(feature)
+
+    if missing:
+        raise ValueError(f"Features not found: {missing}")
+
+    # Rename the DataFrame columns to match the model features
+    return df.rename(columns=rename_dict)
+
+
+def convert_categorical_types(df: pd.DataFrame, features: list, category_mappings={}) -> tuple:
+    """
+    Converts appropriate columns to categorical type with consistent mappings.
+
+    Args:
+        df (pd.DataFrame): The DataFrame to process.
+        features (list): List of feature names to consider for conversion.
+        category_mappings (dict, optional): Existing category mappings. If empty dict, we're in
+                                            training mode. If populated, we're in inference mode.
+
+    Returns:
+        tuple: (processed DataFrame, category mappings dictionary)
+    """
+    # Training mode
+    if category_mappings == {}:
+        for col in df.select_dtypes(include=["object", "string"]):
+            if col in features and df[col].nunique() < 20:
+                print(f"Training mode: Converting {col} to category")
+                df[col] = df[col].astype("category")
+                category_mappings[col] = df[col].cat.categories.tolist()  # Store category mappings
+
+    # Inference mode
+    else:
+        for col, categories in category_mappings.items():
+            if col in df.columns:
+                print(f"Inference mode: Applying categorical mapping for {col}")
+                df[col] = pd.Categorical(df[col], categories=categories)  # Apply consistent categorical mapping
+
+    return df, category_mappings
+
+
+def decompress_features(
+    df: pd.DataFrame, features: List[str], compressed_features: List[str]
+) -> Tuple[pd.DataFrame, List[str]]:
+    """Prepare features for the model by decompressing bitstring features
+
+    Args:
+        df (pd.DataFrame): The features DataFrame
+        features (List[str]): Full list of feature names
+        compressed_features (List[str]): List of feature names to decompress (bitstrings)
+
+    Returns:
+        pd.DataFrame: DataFrame with the decompressed features
+        List[str]: Updated list of feature names after decompression
+
+    Raises:
+        ValueError: If any missing values are found in the specified features
+    """
+
+    # Check for any missing values in the required features
+    missing_counts = df[features].isna().sum()
+    if missing_counts.any():
+        missing_features = missing_counts[missing_counts > 0]
+        print(
+            f"WARNING: Found missing values in features: {missing_features.to_dict()}. "
+            "WARNING: You might want to remove/replace all NaN values before processing."
+        )
+
+    # Decompress the specified compressed features
+    decompressed_features = features.copy()
+    for feature in compressed_features:
+        if (feature not in df.columns) or (feature not in features):
+            print(f"Feature '{feature}' not in the features list, skipping decompression.")
+            continue
+
+        # Remove the feature from the list of features to avoid duplication
+        decompressed_features.remove(feature)
+
+        # Handle all compressed features as bitstrings
+        bit_matrix = np.array([list(bitstring) for bitstring in df[feature]], dtype=np.uint8)
+        prefix = feature[:3]
+
+        # Create all new columns at once - avoids fragmentation
+        new_col_names = [f"{prefix}_{i}" for i in range(bit_matrix.shape[1])]
+        new_df = pd.DataFrame(bit_matrix, columns=new_col_names, index=df.index)
+
+        # Add to features list
+        decompressed_features.extend(new_col_names)
+
+        # Drop original column and concatenate new ones
+        df = df.drop(columns=[feature])
+        df = pd.concat([df, new_df], axis=1)
+
+    return df, decompressed_features
+
+
+if __name__ == "__main__":
+    # Template Parameters
+    target = TEMPLATE_PARAMS["target"]
+    features = TEMPLATE_PARAMS["features"]
+    orig_features = features.copy()
+    compressed_features = TEMPLATE_PARAMS["compressed_features"]
+    train_all_data = TEMPLATE_PARAMS["train_all_data"]
+    hyperparameters = TEMPLATE_PARAMS["hyperparameters"]
+    validation_split = 0.2
+
+    # Script arguments for input/output directories
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-dir", type=str, default=os.environ.get("SM_MODEL_DIR", "/opt/ml/model"))
+    parser.add_argument("--train", type=str, default=os.environ.get("SM_CHANNEL_TRAIN", "/opt/ml/input/data/train"))
+    parser.add_argument(
+        "--output-data-dir", type=str, default=os.environ.get("SM_OUTPUT_DATA_DIR", "/opt/ml/output/data")
+    )
+    args = parser.parse_args()
+
+    # Read the training data into DataFrames
+    training_files = [os.path.join(args.train, file) for file in os.listdir(args.train) if file.endswith(".csv")]
+    print(f"Training Files: {training_files}")
+
+    # Combine files and read them all into a single pandas dataframe
+    all_df = pd.concat([pd.read_csv(file, engine="python") for file in training_files])
+
+    # Check if the dataframe is empty
+    check_dataframe(all_df, "training_df")
+
+    # Features/Target output
+    print(f"Target: {target}")
+    print(f"Features: {str(features)}")
+
+    # Convert any features that might be categorical to 'category' type
+    all_df, category_mappings = convert_categorical_types(all_df, features)
+
+    # If we have compressed features, decompress them
+    if compressed_features:
+        print(f"Decompressing features {compressed_features}...")
+        all_df, features = decompress_features(all_df, features, compressed_features)
+
+    # Do we want to train on all the data?
+    if train_all_data:
+        print("Training on ALL of the data")
+        df_train = all_df.copy()
+        df_val = all_df.copy()
+
+    # Does the dataframe have a training column?
+    elif "training" in all_df.columns:
+        print("Found training column, splitting data based on training column")
+        df_train = all_df[all_df["training"]]
+        df_val = all_df[~all_df["training"]]
+    else:
+        # Just do a random training Split
+        print("WARNING: No training column found, splitting data with random state=42")
+        df_train, df_val = train_test_split(all_df, test_size=validation_split, random_state=42)
+    print(f"FIT/TRAIN: {df_train.shape}")
+    print(f"VALIDATION: {df_val.shape}")
+
+    # Prepare features and targets for training
+    X_train = df_train[features]
+    X_validate = df_val[features]
+    y_train = df_train[target]
+    y_validate = df_val[target]
+
+    # Train XGBoost for point predictions
+    print("\nTraining XGBoost for point predictions...")
+    print(f"  Hyperparameters: {hyperparameters}")
+    xgb_model = XGBRegressor(enable_categorical=True, **hyperparameters)
+    xgb_model.fit(X_train, y_train)
+
+    # Evaluate XGBoost performance
+    y_pred_xgb = xgb_model.predict(X_validate)
+    xgb_rmse = root_mean_squared_error(y_validate, y_pred_xgb)
+    xgb_mae = mean_absolute_error(y_validate, y_pred_xgb)
+    xgb_r2 = r2_score(y_validate, y_pred_xgb)
+
+    print(f"\nXGBoost Point Prediction Performance:")
+    print(f"RMSE: {xgb_rmse:.3f}")
+    print(f"MAE: {xgb_mae:.3f}")
+    print(f"R2: {xgb_r2:.3f}")
+
+    # Define confidence levels we want to model
+    confidence_levels = [0.50, 0.68, 0.80, 0.90, 0.95]  # 50%, 68%, 80%, 90%, 95% confidence intervals
+
+    # Store MAPIE models for each confidence level
+    mapie_models = {}
+
+    # Train models for each confidence level
+    for confidence_level in confidence_levels:
+        alpha = 1 - confidence_level
+        lower_q = alpha / 2
+        upper_q = 1 - alpha / 2
+
+        print(f"\nTraining quantile models for {confidence_level * 100:.0f}% confidence interval...")
+        print(f"  Quantiles: {lower_q:.3f}, {upper_q:.3f}, 0.500")
+
+        # Train three models for this confidence level
+        quantile_estimators = []
+        for q in [lower_q, upper_q, 0.5]:
+            print(f"    Training model for quantile {q:.3f}...")
+            est = LGBMRegressor(
+                objective="quantile",
+                alpha=q,
+                n_estimators=1000,
+                max_depth=6,
+                learning_rate=0.01,
+                num_leaves=31,
+                min_child_samples=20,
+                subsample=0.8,
+                colsample_bytree=0.8,
+                random_state=42,
+                verbose=-1,
+                force_col_wise=True,
+            )
+            est.fit(X_train, y_train)
+            quantile_estimators.append(est)
+
+        # Create MAPIE CQR model for this confidence level
+        print(f"  Setting up MAPIE CQR for {confidence_level * 100:.0f}% confidence...")
+        mapie_model = ConformalizedQuantileRegressor(
+            quantile_estimators, confidence_level=confidence_level, prefit=True
+        )
+
+        # Conformalize the model
+        print(f"  Conformalizing with validation data...")
+        mapie_model.conformalize(X_validate, y_validate)
+
+        # Store the model
+        mapie_models[f"mapie_{confidence_level:.2f}"] = mapie_model
+
+        # Validate coverage for this confidence level
+        y_pred, y_pis = mapie_model.predict_interval(X_validate)
+        coverage = np.mean((y_validate >= y_pis[:, 0, 0]) & (y_validate <= y_pis[:, 1, 0]))
+        print(f"  Coverage: Target={confidence_level * 100:.0f}%, Empirical={coverage * 100:.1f}%")
+
+    print(f"\nOverall Model Performance Summary:")
+    print(f"XGBoost RMSE: {xgb_rmse:.3f}")
+    print(f"XGBoost MAE: {xgb_mae:.3f}")
+    print(f"XGBoost R2: {xgb_r2:.3f}")
+    print(f"NumRows: {len(df_val)}")
+
+    # Analyze interval widths across confidence levels
+    print(f"\nInterval Width Analysis:")
+    for conf_level in confidence_levels:
+        model = mapie_models[f"mapie_{conf_level:.2f}"]
+        _, y_pis = model.predict_interval(X_validate)
+        widths = y_pis[:, 1, 0] - y_pis[:, 0, 0]
+        print(f"  {conf_level * 100:.0f}% CI: Mean width={np.mean(widths):.3f}, Std={np.std(widths):.3f}")
+
+    # Save the trained XGBoost model
+    joblib.dump(xgb_model, os.path.join(args.model_dir, "xgb_model.joblib"))
+
+    # Save all MAPIE models
+    for model_name, model in mapie_models.items():
+        joblib.dump(model, os.path.join(args.model_dir, f"{model_name}.joblib"))
+
+    # Save the feature list
+    with open(os.path.join(args.model_dir, "feature_columns.json"), "w") as fp:
+        json.dump(features, fp)
+
+    # Save category mappings if any
+    if category_mappings:
+        with open(os.path.join(args.model_dir, "category_mappings.json"), "w") as fp:
+            json.dump(category_mappings, fp)
+
+    # Save model configuration
+    model_config = {
+        "model_type": "XGBoost_MAPIE_CQR_LightGBM",
+        "confidence_levels": confidence_levels,
+        "n_features": len(features),
+        "target": target,
+        "validation_metrics": {
+            "xgb_rmse": float(xgb_rmse),
+            "xgb_mae": float(xgb_mae),
+            "xgb_r2": float(xgb_r2),
+            "n_validation": len(df_val),
+        },
+    }
+    with open(os.path.join(args.model_dir, "model_config.json"), "w") as fp:
+        json.dump(model_config, fp, indent=2)
+
+    print(f"\nModel training complete!")
+    print(f"Saved 1 XGBoost model and {len(mapie_models)} MAPIE models to {args.model_dir}")
+
+
+#
+# Inference Section
+#
+def model_fn(model_dir) -> dict:
+    """Load XGBoost and all MAPIE models from the specified directory."""
+
+    # Load model configuration to know which models to load
+    with open(os.path.join(model_dir, "model_config.json")) as fp:
+        config = json.load(fp)
+
+    # Load XGBoost regressor
+    xgb_path = os.path.join(model_dir, "xgb_model.joblib")
+    xgb_model = joblib.load(xgb_path)
+
+    # Load all MAPIE models
+    mapie_models = {}
+    for conf_level in config["confidence_levels"]:
+        model_name = f"mapie_{conf_level:.2f}"
+        mapie_models[model_name] = joblib.load(os.path.join(model_dir, f"{model_name}.joblib"))
+
+    # Load category mappings if they exist
+    category_mappings = {}
+    category_path = os.path.join(model_dir, "category_mappings.json")
+    if os.path.exists(category_path):
+        with open(category_path) as fp:
+            category_mappings = json.load(fp)
+
+    return {
+        "xgb_model": xgb_model,
+        "mapie_models": mapie_models,
+        "confidence_levels": config["confidence_levels"],
+        "category_mappings": category_mappings,
+    }
+
+
+def input_fn(input_data, content_type):
+    """Parse input data and return a DataFrame."""
+    if not input_data:
+        raise ValueError("Empty input data is not supported!")
+
+    # Decode bytes to string if necessary
+    if isinstance(input_data, bytes):
+        input_data = input_data.decode("utf-8")
+
+    if "text/csv" in content_type:
+        return pd.read_csv(StringIO(input_data))
+    elif "application/json" in content_type:
+        return pd.DataFrame(json.loads(input_data))
+    else:
+        raise ValueError(f"{content_type} not supported!")
+
+
+def output_fn(output_df, accept_type):
+    """Supports both CSV and JSON output formats."""
+    if "text/csv" in accept_type:
+        # Convert categorical columns to string to avoid fillna issues
+        for col in output_df.select_dtypes(include=["category"]).columns:
+            output_df[col] = output_df[col].astype(str)
+        csv_output = output_df.fillna("N/A").to_csv(index=False)
+        return csv_output, "text/csv"
+    elif "application/json" in accept_type:
+        return output_df.to_json(orient="records"), "application/json"
+    else:
+        raise RuntimeError(f"{accept_type} accept type is not supported by this script.")
+
+
+def predict_fn(df, models) -> pd.DataFrame:
+    """Make predictions using XGBoost for point estimates and MAPIE for conformalized intervals
+
+    Args:
+        df (pd.DataFrame): The input DataFrame
+        models (dict): Dictionary containing XGBoost and MAPIE models
+
+    Returns:
+        pd.DataFrame: DataFrame with XGBoost predictions and conformalized intervals
+    """
+
+    # Flag for outlier stretch adjustment for the prediction intervals
+    # if the predicted values are outside the intervals
+    outlier_stretch = False
+
+    # Grab our feature columns (from training)
+    model_dir = os.environ.get("SM_MODEL_DIR", "/opt/ml/model")
+    with open(os.path.join(model_dir, "feature_columns.json")) as fp:
+        model_features = json.load(fp)
+
+    # Match features in a case-insensitive manner
+    matched_df = match_features_case_insensitive(df, model_features)
+
+    # Apply categorical mappings if they exist
+    if models.get("category_mappings"):
+        matched_df, _ = convert_categorical_types(matched_df, model_features, models["category_mappings"])
+
+    # Get features for prediction
+    X = matched_df[model_features]
+
+    # Get XGBoost point predictions
+    df["prediction"] = models["xgb_model"].predict(X)
+
+    # Get predictions from each MAPIE model for conformalized intervals
+    for conf_level in models["confidence_levels"]:
+        model_name = f"mapie_{conf_level:.2f}"
+        model = models["mapie_models"][model_name]
+
+        # Get conformalized predictions
+        y_pred, y_pis = model.predict_interval(X)
+
+        # Map confidence levels to quantile names
+        if conf_level == 0.50:  # 50% CI
+            df["q_25"] = y_pis[:, 0, 0]
+            df["q_75"] = y_pis[:, 1, 0]
+        elif conf_level == 0.68:  # 68% CI
+            df["q_16"] = y_pis[:, 0, 0]
+            df["q_84"] = y_pis[:, 1, 0]
+        elif conf_level == 0.80:  # 80% CI
+            df["q_10"] = y_pis[:, 0, 0]
+            df["q_90"] = y_pis[:, 1, 0]
+        elif conf_level == 0.90:  # 90% CI
+            df["q_05"] = y_pis[:, 0, 0]
+            df["q_95"] = y_pis[:, 1, 0]
+        elif conf_level == 0.95:  # 95% CI
+            df["q_025"] = y_pis[:, 0, 0]
+            df["q_975"] = y_pis[:, 1, 0]
+
+    # Add median (q_50) from XGBoost prediction
+    df["q_50"] = df["prediction"]
+
+    # Calculate a pseudo-standard deviation from the 68% interval width
+    df["prediction_std"] = (df["q_84"] - df["q_16"]) / 2.0
+
+    # Reorder the quantile columns for easier reading
+    quantile_cols = ["q_025", "q_05", "q_10", "q_16", "q_25", "q_75", "q_84", "q_90", "q_95", "q_975"]
+    other_cols = [col for col in df.columns if col not in quantile_cols]
+    df = df[other_cols + quantile_cols]
+
+    # Adjust the outer quantiles to ensure they encompass the prediction
+    if outlier_stretch:
+        # Lower intervals adjustments
+        df["q_025"] = np.minimum(df["q_025"], df["prediction"])
+        df["q_05"] = np.minimum(df["q_05"], df["prediction"])
+        df["q_10"] = np.minimum(df["q_10"], df["prediction"])
+        df["q_16"] = np.minimum(df["q_16"], df["prediction"])
+        df["q_25"] = np.minimum(df["q_25"], df["prediction"])
+
+        # Upper intervals adjustments
+        df["q_75"] = np.maximum(df["q_75"], df["prediction"])
+        df["q_84"] = np.maximum(df["q_84"], df["prediction"])
+        df["q_90"] = np.maximum(df["q_90"], df["prediction"])
+        df["q_95"] = np.maximum(df["q_95"], df["prediction"])
+        df["q_975"] = np.maximum(df["q_975"], df["prediction"])
+
+    return df

workbench/model_scripts/{custom_models/uq_models → uq_models}/mapie.template

@@ -303,7 +303,7 @@ if __name__ == "__main__":
         print(f"  {conf_level * 100:.0f}% CI: Mean width={np.mean(widths):.3f}, Std={np.std(widths):.3f}")
 
     # Save the trained XGBoost model
-    xgb_model.save_model(os.path.join(args.model_dir, "xgb_model.json"))
+    joblib.dump(xgb_model, os.path.join(args.model_dir, "xgb_model.joblib"))
 
     # Save all MAPIE models
     for model_name, model in mapie_models.items():
@@ -349,9 +349,8 @@ def model_fn(model_dir) -> dict:
         config = json.load(fp)
 
     # Load XGBoost regressor
-    xgb_path = os.path.join(model_dir, "xgb_model.json")
-    xgb_model = XGBRegressor(enable_categorical=True)
-    xgb_model.load_model(xgb_path)
+    xgb_path = os.path.join(model_dir, "xgb_model.joblib")
+    xgb_model = joblib.load(xgb_path)
 
     # Load all MAPIE models
     mapie_models = {}

workbench/model_scripts/uq_models/requirements.txt

@@ -0,0 +1 @@
+# Note: Most libs are already in the training/inference images, ONLY specify additional libs here

workbench/model_scripts/xgb_model/xgb_model.template

@@ -145,7 +145,7 @@ def convert_categorical_types(df: pd.DataFrame, features: list, category_mapping
 def decompress_features(
     df: pd.DataFrame, features: List[str], compressed_features: List[str]
 ) -> Tuple[pd.DataFrame, List[str]]:
-    """Prepare features for the model
+    """Prepare features for the model by decompressing bitstring features
 
     Args:
         df (pd.DataFrame): The features DataFrame
@@ -170,7 +170,7 @@ def decompress_features(
         )
 
     # Decompress the specified compressed features
-    decompressed_features = features
+    decompressed_features = features.copy()
     for feature in compressed_features:
         if (feature not in df.columns) or (feature not in features):
             print(f"Feature '{feature}' not in the features list, skipping decompression.")

workbench/repl/workbench_shell.py

@@ -560,7 +560,7 @@ class WorkbenchShell:
         from workbench.web_interface.components.plugin_unit_test import PluginUnitTest
 
         # Get kwargs
-        theme = kwargs.get("theme", "dark")
+        theme = kwargs.get("theme", "midnight_blue")
 
         plugin_test = PluginUnitTest(plugin_class, theme=theme, input_data=data, **kwargs)
 

workbench/utils/model_utils.py

@@ -3,6 +3,7 @@
 import logging
 import pandas as pd
 import numpy as np
+from scipy.stats import spearmanr
 import importlib.resources
 from pathlib import Path
 import os
@@ -222,8 +223,8 @@ def uq_metrics(df: pd.DataFrame, target_col: str) -> Dict[str, Any]:
         lower_95, upper_95 = df["q_025"], df["q_975"]
         lower_90, upper_90 = df["q_05"], df["q_95"]
         lower_80, upper_80 = df["q_10"], df["q_90"]
-        lower_68 = df.get("q_16", 0)
-        upper_68 = df.get("q_84", 0)
+        lower_68 = df.get("q_16", df["q_10"])  # fallback to 80% interval
+        upper_68 = df.get("q_84", df["q_90"])  # fallback to 80% interval
         lower_50, upper_50 = df["q_25"], df["q_75"]
     elif "prediction_std" in df.columns:
         lower_95 = df["prediction"] - 1.96 * df["prediction_std"]
@@ -240,18 +241,16 @@ def uq_metrics(df: pd.DataFrame, target_col: str) -> Dict[str, Any]:
         raise ValueError(
             "Either quantile columns (q_025, q_975, q_25, q_75) or 'prediction_std' column must be present."
         )
-    avg_std = df["prediction_std"].mean()
     median_std = df["prediction_std"].median()
     coverage_95 = np.mean((df[target_col] >= lower_95) & (df[target_col] <= upper_95))
     coverage_90 = np.mean((df[target_col] >= lower_90) & (df[target_col] <= upper_90))
     coverage_80 = np.mean((df[target_col] >= lower_80) & (df[target_col] <= upper_80))
     coverage_68 = np.mean((df[target_col] >= lower_68) & (df[target_col] <= upper_68))
-    coverage_50 = np.mean((df[target_col] >= lower_50) & (df[target_col] <= upper_50))
-    avg_width_95 = np.mean(upper_95 - lower_95)
-    avg_width_90 = np.mean(upper_90 - lower_90)
-    avg_width_80 = np.mean(upper_80 - lower_80)
-    avg_width_50 = np.mean(upper_50 - lower_50)
-    avg_width_68 = np.mean(upper_68 - lower_68)
+    median_width_95 = np.median(upper_95 - lower_95)
+    median_width_90 = np.median(upper_90 - lower_90)
+    median_width_80 = np.median(upper_80 - lower_80)
+    median_width_50 = np.median(upper_50 - lower_50)
+    median_width_68 = np.median(upper_68 - lower_68)
 
     # --- CRPS (measures calibration + sharpness) ---
     z = (df[target_col] - df["prediction"]) / df["prediction_std"]
@@ -267,47 +266,43 @@ def uq_metrics(df: pd.DataFrame, target_col: str) -> Dict[str, Any]:
     )
     mean_is_95 = np.mean(is_95)
 
-    # --- Adaptive Calibration (correlation between errors and uncertainty) ---
+    # --- Interval to Error Correlation ---
     abs_residuals = np.abs(df[target_col] - df["prediction"])
-    width_95 = upper_95 - lower_95
-    adaptive_calibration = np.corrcoef(abs_residuals, width_95)[0, 1]
+    width_68 = upper_68 - lower_68
+
+    # Spearman correlation for robustness
+    interval_to_error_corr = spearmanr(width_68, abs_residuals)[0]
 
     # Collect results
     results = {
-        "coverage_50": coverage_50,
         "coverage_68": coverage_68,
         "coverage_80": coverage_80,
         "coverage_90": coverage_90,
         "coverage_95": coverage_95,
         "median_std": median_std,
-        "avg_std": avg_std,
-        "avg_width_50": avg_width_50,
-        "avg_width_68": avg_width_68,
-        "avg_width_80": avg_width_80,
-        "avg_width_90": avg_width_90,
-        "avg_width_95": avg_width_95,
-        # "crps": mean_crps,
-        # "interval_score_95": mean_is_95,
-        # "adaptive_calibration": adaptive_calibration,
+        "median_width_50": median_width_50,
+        "median_width_68": median_width_68,
+        "median_width_80": median_width_80,
+        "median_width_90": median_width_90,
+        "median_width_95": median_width_95,
+        "interval_to_error_corr": interval_to_error_corr,
         "n_samples": len(df),
     }
 
     print("\n=== UQ Metrics ===")
-    print(f"Coverage @ 50%: {coverage_50:.3f} (target: 0.50)")
     print(f"Coverage @ 68%: {coverage_68:.3f} (target: 0.68)")
     print(f"Coverage @ 80%: {coverage_80:.3f} (target: 0.80)")
     print(f"Coverage @ 90%: {coverage_90:.3f} (target: 0.90)")
     print(f"Coverage @ 95%: {coverage_95:.3f} (target: 0.95)")
     print(f"Median Prediction StdDev: {median_std:.3f}")
-    print(f"Avg Prediction StdDev: {avg_std:.3f}")
-    print(f"Average 50% Width: {avg_width_50:.3f}")
-    print(f"Average 68% Width: {avg_width_68:.3f}")
-    print(f"Average 80% Width: {avg_width_80:.3f}")
-    print(f"Average 90% Width: {avg_width_90:.3f}")
-    print(f"Average 95% Width: {avg_width_95:.3f}")
+    print(f"Median 50% Width: {median_width_50:.3f}")
+    print(f"Median 68% Width: {median_width_68:.3f}")
+    print(f"Median 80% Width: {median_width_80:.3f}")
+    print(f"Median 90% Width: {median_width_90:.3f}")
+    print(f"Median 95% Width: {median_width_95:.3f}")
     print(f"CRPS: {mean_crps:.3f} (lower is better)")
     print(f"Interval Score 95%: {mean_is_95:.3f} (lower is better)")
-    print(f"Adaptive Calibration: {adaptive_calibration:.3f} (higher is better, target: >0.5)")
+    print(f"Interval/Error Corr: {interval_to_error_corr:.3f} (higher is better, target: >0.5)")
     print(f"Samples: {len(df)}")
     return results
 

workbench/web_interface/components/plugins/scatter_plot.py

@@ -159,7 +159,7 @@ class ScatterPlot(PluginInterface):
         self.df = self.df.drop(columns=aws_cols, errors="ignore")
 
         # Set hover columns and custom data
-        self.hover_columns = kwargs.get("hover_columns", self.df.columns.tolist()[:10])
+        self.hover_columns = kwargs.get("hover_columns", sorted(self.df.columns.tolist()[:15]))
         self.suppress_hover_display = kwargs.get("suppress_hover_display", False)
         self.custom_data = kwargs.get("custom_data", [])
 
@@ -427,7 +427,7 @@ if __name__ == "__main__":
 
     from workbench.api import DFStore
 
-    df = DFStore().get("/workbench/models/aqsol-uq/auto_inference")
+    df = DFStore().get("/workbench/models/aqsol-uq-100/full_cross_fold_inference")
 
     # Run the Unit Test on the Plugin
     PluginUnitTest(
@@ -436,6 +436,6 @@ if __name__ == "__main__":
         theme="midnight_blue",
         x="solubility",
         y="prediction",
-        color="residuals_abs",
+        color="prediction_std",
         suppress_hover_display=True,
     ).run()

{workbench-0.8.180.dist-info → workbench-0.8.182.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: workbench
-Version: 0.8.180
+Version: 0.8.182
 Summary: Workbench: A Dashboard and Python API for creating and deploying AWS SageMaker Model Pipelines
 Author-email: SuperCowPowers LLC <support@supercowpowers.com>
 License-Expression: MIT

{workbench-0.8.180.dist-info → workbench-0.8.182.dist-info}/RECORD

@@ -54,7 +54,7 @@ workbench/core/artifacts/cached_artifact_mixin.py,sha256=ngqFLZ4cQx_TFouXZgXZQsv
 workbench/core/artifacts/data_capture_core.py,sha256=q8f79rRTYiZ7T4IQRWXl8ZvPpcvZyNxYERwvo8o0OQc,14858
 workbench/core/artifacts/data_source_abstract.py,sha256=5IRCzFVK-17cd4NXPMRfx99vQAmQ0WHE5jcm5RfsVTg,10619
 workbench/core/artifacts/data_source_factory.py,sha256=YL_tA5fsgubbB3dPF6T4tO0rGgz-6oo3ge4i_YXVC-M,2380
-workbench/core/artifacts/endpoint_core.py,sha256=S9QHtXrQXkCTY-coaYSsAGJi60nZj3Aq9ruyVE1PUQs,49224
+workbench/core/artifacts/endpoint_core.py,sha256=b3cNj1UnlHmQdG1C8bmD2jWpD4h-O6F-75fWSm01uGU,51850
 workbench/core/artifacts/feature_set_core.py,sha256=7b1o_PzxtwaYC-W2zxlkltiO0fYULA8CVGWwHNmqgtI,31457
 workbench/core/artifacts/model_core.py,sha256=ECDwQ0qM5qb1yGJ07U70BVdfkrW9m7p9e6YJWib3uR0,50855
 workbench/core/artifacts/monitor_core.py,sha256=M307yz7tEzOEHgv-LmtVy9jKjSbM98fHW3ckmNYrwlU,27897
@@ -122,7 +122,7 @@ workbench/core/views/training_view.py,sha256=UWW8Asxtm_kV7Z8NooitMA4xC5vTc7lSWwT
 workbench/core/views/view.py,sha256=Ujzw6zLROP9oKfKm3zJwaOyfpyjh5uM9fAu1i3kUOig,11764
 workbench/core/views/view_utils.py,sha256=y0YuPW-90nAfgAD1UW_49-j7Mvncfm7-5rV8I_97CK8,12274
 workbench/core/views/storage/mdq_view.py,sha256=qf_ep1KwaXOIfO930laEwNIiCYP7VNOqjE3VdHfopRE,5195
-workbench/model_scripts/script_generation.py,sha256=dL23XYwEsHIStc7i53DtF_47FqOrI9gq0kQAT6sNpZ8,7923
+workbench/model_scripts/script_generation.py,sha256=dLxVRrvrrI_HQatJRAXta6UEbFFbkgITNvDJllQZyCM,7905
 workbench/model_scripts/custom_models/chem_info/Readme.md,sha256=mH1lxJ4Pb7F5nBnVXaiuxpi8zS_yjUw_LBJepVKXhlA,574
 workbench/model_scripts/custom_models/chem_info/mol_descriptors.py,sha256=c8gkHZ-8s3HJaW9zN9pnYGK7YVW8Y0xFqQ1G_ysrF2Y,18789
 workbench/model_scripts/custom_models/chem_info/mol_standardize.py,sha256=qPLCdVMSXMOWN-01O1isg2zq7eQyFAI0SNatHkRq1uw,17524
@@ -140,8 +140,7 @@ workbench/model_scripts/custom_models/uq_models/Readme.md,sha256=UVpL-lvtTrLqwBe
 workbench/model_scripts/custom_models/uq_models/bayesian_ridge.template,sha256=ca3CaAk6HVuNv1HnPgABTzRY3oDrRxomjgD4V1ZDwoc,6448
 workbench/model_scripts/custom_models/uq_models/ensemble_xgb.template,sha256=xlKLHeLQkScONnrlbAGIsrCm2wwsvcfv4Vdrw4nlc_8,13457
 workbench/model_scripts/custom_models/uq_models/gaussian_process.template,sha256=3nMlCi8nEbc4N-MQTzjfIcljfDQkUmWeLBfmd18m5fg,6632
-workbench/model_scripts/custom_models/uq_models/generated_model_script.py,sha256=hE54OgIsR20-5RBjqqjUp5Z_aMzrg1Ykhwx4J3kG8Pw,19401
-workbench/model_scripts/custom_models/uq_models/mapie.template,sha256=FEBYxM-gVzwHWYga11DZf3ORUqmCrT0eQk8apixwx_E,19000
+workbench/model_scripts/custom_models/uq_models/generated_model_script.py,sha256=Y89qD3gJ8wx9klXXDUQNfoLTImVFcdYLfRz-SA8mppE,21461
 workbench/model_scripts/custom_models/uq_models/meta_uq.template,sha256=XTfhODRaHlI1jZGo9pSe-TqNsk2_nuSw0xMO2fKzDv8,14011
 workbench/model_scripts/custom_models/uq_models/ngboost.template,sha256=v1rviYTJGJnQRGgAyveXhOQlS-WFCTlc2vdnWq6HIXk,8241
 workbench/model_scripts/custom_models/uq_models/proximity.py,sha256=zqmNlX70LnWXr5fdtFFQppSNTLjlOciQVrjGr-g9jRE,13716
@@ -154,16 +153,17 @@ workbench/model_scripts/ensemble_xgb/requirements.txt,sha256=jWlGc7HH7vqyukTm38L
 workbench/model_scripts/pytorch_model/generated_model_script.py,sha256=Mr1IMQJE_ML899qjzhjkrP521IjvcAvqU0pk--FB7KY,22356
 workbench/model_scripts/pytorch_model/pytorch.template,sha256=_gRp6DH294FLxF21UpSTq7s9RFfrLjViKvjXQ4yDfBQ,21999
 workbench/model_scripts/pytorch_model/requirements.txt,sha256=ICS5nW0wix44EJO2tJszJSaUrSvhSfdedn6FcRInGx4,181
-workbench/model_scripts/quant_regression/quant_regression.template,sha256=2F25lZ7m_VafHvuGrC__R3uB1NzKgZu94eWJS9sWpYg,9783
-workbench/model_scripts/quant_regression/requirements.txt,sha256=jWlGc7HH7vqyukTm38LN4EyDi8jDUPEay4n45z-30uc,104
 workbench/model_scripts/scikit_learn/generated_model_script.py,sha256=c73ZpJBlU5k13Nx-ZDkLXu7da40CYyhwjwwmuPq6uLg,12870
 workbench/model_scripts/scikit_learn/requirements.txt,sha256=aVvwiJ3LgBUhM_PyFlb2gHXu_kpGPho3ANBzlOkfcvs,107
 workbench/model_scripts/scikit_learn/scikit_learn.template,sha256=QQvqx-eX9ZTbYmyupq6R6vIQwosmsmY_MRBPaHyfjdk,12586
+workbench/model_scripts/uq_models/generated_model_script.py,sha256=Y89qD3gJ8wx9klXXDUQNfoLTImVFcdYLfRz-SA8mppE,21461
+workbench/model_scripts/uq_models/mapie.template,sha256=8VzoP-Wp3ECVIDqXVkiTS6bwmn3cd3dDZ2WjYPzXTi8,18955
+workbench/model_scripts/uq_models/requirements.txt,sha256=fw7T7t_YJAXK3T6Ysbesxh_Agx_tv0oYx72cEBTqRDY,98
 workbench/model_scripts/xgb_model/generated_model_script.py,sha256=Tbn7EMXxZZO8rDdKQ5fYCbpltACsMXNvuusLL9p-U5c,22319
 workbench/model_scripts/xgb_model/requirements.txt,sha256=jWlGc7HH7vqyukTm38LN4EyDi8jDUPEay4n45z-30uc,104
-workbench/model_scripts/xgb_model/xgb_model.template,sha256=9wed0Ii6KZpEiVj6fHGDkwpxPkLIhcSIDls7Ij0HUF0,17874
+workbench/model_scripts/xgb_model/xgb_model.template,sha256=0uXknIEqgUaIFUfu2gfkxa3WHUr8HBBqBepGUTDvrhQ,17917
 workbench/repl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-workbench/repl/workbench_shell.py,sha256=Vhg4BQr2r4D4ymekrVtOFi0MaRvaH4V2UgcWRgvN_3U,22122
+workbench/repl/workbench_shell.py,sha256=Gw7-tWCc6k7nwWrEimg-YuQOnWCsOzc2QMiX3Rt-Wpk,22131
 workbench/resources/open_source_api.key,sha256=3S0OTblsmC0msUPdE_dbBmI83xJNmYscuwLJ57JmuOc,433
 workbench/resources/signature_verify_pub.pem,sha256=V3-u-3_z2PH-805ybkKvzDOBwAbvHxcKn0jLBImEtzM,272
 workbench/scripts/check_double_bond_stereo.py,sha256=p5hnL54Weq77ES0HCELq9JeoM-PyUGkvVSeWYF2dKyo,7776
@@ -219,7 +219,7 @@ workbench/utils/lambda_utils.py,sha256=7GhGRPyXn9o-toWb9HBGSnI8-DhK9YRkwhCSk_mNK
 workbench/utils/license_manager.py,sha256=sDuhk1mZZqUbFmnuFXehyGnui_ALxrmYBg7gYwoo7ho,6975
 workbench/utils/log_utils.py,sha256=7n1NJXO_jUX82e6LWAQug6oPo3wiPDBYsqk9gsYab_A,3167
 workbench/utils/markdown_utils.py,sha256=4lEqzgG4EVmLcvvKKNUwNxVCySLQKJTJmWDiaDroI1w,8306
-workbench/utils/model_utils.py,sha256=b71URPIsbkmFy2XEglftO47lQGIl5CpahjCbb2CnTpM,12801
+workbench/utils/model_utils.py,sha256=sbsWDaeTagHdrh8Z_A_yfWsiSIEqSl18sq67l9E2xiA,12602
 workbench/utils/monitor_utils.py,sha256=kVaJ7BgUXs3VPMFYfLC03wkIV4Dq-pEhoXS0wkJFxCc,7858
 workbench/utils/pandas_utils.py,sha256=uTUx-d1KYfjbS9PMQp2_9FogCV7xVZR6XLzU5YAGmfs,39371
 workbench/utils/performance_utils.py,sha256=WDNvz-bOdC99cDuXl0urAV4DJ7alk_V3yzKPwvqgST4,1329
@@ -278,7 +278,7 @@ workbench/web_interface/components/plugins/molecule_panel.py,sha256=xGCEI5af8F5l
 workbench/web_interface/components/plugins/molecule_viewer.py,sha256=xavixcu4RNzh6Nj_-3-XlK09DgpNx5jGmo3wEPNftiE,4529
 workbench/web_interface/components/plugins/pipeline_details.py,sha256=caiFIakHk-1dGGNW7wlio2X7iAm2_tCNbSjDzoRWGEk,5534
 workbench/web_interface/components/plugins/proximity_mini_graph.py,sha256=b_YYnvLczJUhaDbrrXnyjUDYF7C4R4ufCZXtJiyRnJ0,7233
-workbench/web_interface/components/plugins/scatter_plot.py,sha256=j8J1-m_xZjG0hgaMevbRvKaTAze0GglpMMDlP3WA_6U,19106
+workbench/web_interface/components/plugins/scatter_plot.py,sha256=8tYnHlgi2UnuKLoxi9-89QF8ZHFrPhpRkzBY5gzlVdo,19130
 workbench/web_interface/components/plugins/shap_summary_plot.py,sha256=_V-xxVehU-60IpYWvAqTW5x_6u6pbjz9mI8r0ppIXKg,9454
 workbench/web_interface/page_views/data_sources_page_view.py,sha256=SXNUG6n_eP9i4anddEXd5E9rMRt-R2EyNR-bbe8OQK4,4673
 workbench/web_interface/page_views/endpoints_page_view.py,sha256=EI3hA18pEn-mAPEzGAw0W-wM8qJR2j_8pQEJlbJCENk,2770
@@ -287,9 +287,9 @@ workbench/web_interface/page_views/main_page.py,sha256=X4-KyGTKLAdxR-Zk2niuLJB2Y
 workbench/web_interface/page_views/models_page_view.py,sha256=M0bdC7bAzLyIaE2jviY12FF4abdMFZmg6sFuOY_LaGI,2650
 workbench/web_interface/page_views/page_view.py,sha256=Gh6YnpOGlUejx-bHZAf5pzqoQ1H1R0OSwOpGhOBO06w,455
 workbench/web_interface/page_views/pipelines_page_view.py,sha256=v2pxrIbsHBcYiblfius3JK766NZ7ciD2yPx0t3E5IJo,2656
-workbench-0.8.180.dist-info/licenses/LICENSE,sha256=z4QMMPlLJkZjU8VOKqJkZiQZCEZ--saIU2Z8-p3aVc0,1080
-workbench-0.8.180.dist-info/METADATA,sha256=p3RyXhe_7ljNODfTkP6bW44_TQhwIriZnyJsJ41Xtck,9210
-workbench-0.8.180.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-workbench-0.8.180.dist-info/entry_points.txt,sha256=zPFPruY9uayk8-wsKrhfnIyIB6jvZOW_ibyllEIsLWo,356
-workbench-0.8.180.dist-info/top_level.txt,sha256=Dhy72zTxaA_o_yRkPZx5zw-fwumnjGaeGf0hBN3jc_w,10
-workbench-0.8.180.dist-info/RECORD,,
+workbench-0.8.182.dist-info/licenses/LICENSE,sha256=z4QMMPlLJkZjU8VOKqJkZiQZCEZ--saIU2Z8-p3aVc0,1080
+workbench-0.8.182.dist-info/METADATA,sha256=mNhjFYFQS3MP33Z2CCAI2ydB_BPSpTxlvSAQiwTQdy8,9210
+workbench-0.8.182.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+workbench-0.8.182.dist-info/entry_points.txt,sha256=zPFPruY9uayk8-wsKrhfnIyIB6jvZOW_ibyllEIsLWo,356
+workbench-0.8.182.dist-info/top_level.txt,sha256=Dhy72zTxaA_o_yRkPZx5zw-fwumnjGaeGf0hBN3jc_w,10
+workbench-0.8.182.dist-info/RECORD,,

workbench/model_scripts/quant_regression/quant_regression.template

@@ -1,274 +0,0 @@
-# Imports for XGB Model
-import xgboost as xgb
-import awswrangler as wr
-from sklearn.model_selection import train_test_split
-
-# Model Performance Scores
-from sklearn.metrics import mean_absolute_error, r2_score, root_mean_squared_error
-
-from io import StringIO
-import json
-import argparse
-import os
-import pandas as pd
-
-# Template Placeholders
-TEMPLATE_PARAMS = {
-    "model_type": "{{model_type}}",
-    "target_column": "{{target_column}}",
-    "features": "{{feature_list}}",
-    "model_metrics_s3_path": "{{model_metrics_s3_path}}",
-    "train_all_data": "{{train_all_data}}",
-}
-
-
-# Function to check if dataframe is empty
-def check_dataframe(df: pd.DataFrame, df_name: str) -> None:
-    """
-    Check if the provided dataframe is empty and raise an exception if it is.
-
-    Args:
-        df (pd.DataFrame): DataFrame to check
-        df_name (str): Name of the DataFrame
-    """
-    if df.empty:
-        msg = f"*** The training data {df_name} has 0 rows! ***STOPPING***"
-        print(msg)
-        raise ValueError(msg)
-
-
-def match_features_case_insensitive(df: pd.DataFrame, model_features: list) -> pd.DataFrame:
-    """
-    Matches and renames DataFrame columns to match model feature names (case-insensitive).
-    Prioritizes exact matches, then case-insensitive matches.
-
-    Raises ValueError if any model features cannot be matched.
-    """
-    df_columns_lower = {col.lower(): col for col in df.columns}
-    rename_dict = {}
-    missing = []
-    for feature in model_features:
-        if feature in df.columns:
-            continue  # Exact match
-        elif feature.lower() in df_columns_lower:
-            rename_dict[df_columns_lower[feature.lower()]] = feature
-        else:
-            missing.append(feature)
-
-    if missing:
-        raise ValueError(f"Features not found: {missing}")
-
-    # Rename the DataFrame columns to match the model features
-    return df.rename(columns=rename_dict)
-
-
-if __name__ == "__main__":
-    """The main function is for training the XGBoost Quantile Regression models"""
-
-    # Harness Template Parameters
-    target = TEMPLATE_PARAMS["target_column"]
-    features = TEMPLATE_PARAMS["features"]
-    model_metrics_s3_path = TEMPLATE_PARAMS["model_metrics_s3_path"]
-    train_all_data = TEMPLATE_PARAMS["train_all_data"]
-    validation_split = 0.2
-    quantiles = [0.025, 0.25, 0.50, 0.75, 0.975]
-    q_models = {}
-
-    # Script arguments for input/output directories
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--model-dir", type=str, default=os.environ.get("SM_MODEL_DIR", "/opt/ml/model"))
-    parser.add_argument("--train", type=str, default=os.environ.get("SM_CHANNEL_TRAIN", "/opt/ml/input/data/train"))
-    parser.add_argument(
-        "--output-data-dir", type=str, default=os.environ.get("SM_OUTPUT_DATA_DIR", "/opt/ml/output/data")
-    )
-    args = parser.parse_args()
-
-    # Load training data from the specified directory
-    training_files = [os.path.join(args.train, file) for file in os.listdir(args.train) if file.endswith(".csv")]
-    print(f"Training Files: {training_files}")
-
-    # Combine files and read them all into a single pandas dataframe
-    df = pd.concat([pd.read_csv(file, engine="python") for file in training_files])
-
-    # Check if the DataFrame is empty
-    check_dataframe(df, "training_df")
-
-    # Training data split logic
-    if train_all_data:
-        # Use all data for both training and validation
-        print("Training on all data...")
-        df_train = df.copy()
-        df_val = df.copy()
-    elif "training" in df.columns:
-        # Split data based on a 'training' column if it exists
-        print("Splitting data based on 'training' column...")
-        df_train = df[df["training"]].copy()
-        df_val = df[~df["training"]].copy()
-    else:
-        # Perform a random split if no 'training' column is found
-        print("Splitting data randomly...")
-        df_train, df_val = train_test_split(df, test_size=validation_split, random_state=42)
-
-    # Features/Target output
-    print(f"Target: {target}")
-    print(f"Features: {str(features)}")
-    print(f"Data Shape: {df.shape}")
-
-    # Prepare features and targets for training
-    X_train = df_train[features]
-    X_val = df_val[features]
-    y_train = df_train[target]
-    y_val = df_val[target]
-
-    # Train models for each of the quantiles
-    for q in quantiles:
-        params = {
-            "objective": "reg:quantileerror",
-            "quantile_alpha": q,
-        }
-        model = xgb.XGBRegressor(**params)
-        model.fit(X_train, y_train)
-
-        # Convert quantile to string
-        q_str = f"q_{int(q * 100)}" if (q * 100) == int(q * 100) else f"q_{int(q * 1000):03d}"
-
-        # Store the model
-        q_models[q_str] = model
-
-    # Run predictions for each quantile
-    quantile_predictions = {q: model.predict(X_val) for q, model in q_models.items()}
-
-    # Create a copy of the validation DataFrame and add the new columns
-    result_df = df_val[[target]].copy()
-
-    # Add the quantile predictions to the DataFrame
-    for name, preds in quantile_predictions.items():
-        result_df[name] = preds
-
-    # Add the median as the main prediction
-    result_df["prediction"] = result_df["q_50"]
-
-    # Now compute residuals on the prediction
-    result_df["residual"] = result_df[target] - result_df["prediction"]
-    result_df["residual_abs"] = result_df["residual"].abs()
-
-    # Save the results dataframe to S3
-    wr.s3.to_csv(
-        result_df,
-        path=f"{model_metrics_s3_path}/validation_predictions.csv",
-        index=False,
-    )
-
-    # Report Performance Metrics
-    rmse = root_mean_squared_error(result_df[target], result_df["prediction"])
-    mae = mean_absolute_error(result_df[target], result_df["prediction"])
-    r2 = r2_score(result_df[target], result_df["prediction"])
-    print(f"RMSE: {rmse:.3f}")
-    print(f"MAE: {mae:.3f}")
-    print(f"R2: {r2:.3f}")
-    print(f"NumRows: {len(result_df)}")
-
-    # Now save the quantile models
-    for name, model in q_models.items():
-        model_path = os.path.join(args.model_dir, f"{name}.json")
-        print(f"Saving model:  {model_path}")
-        model.save_model(model_path)
-
-    # Also save the features (this will validate input during predictions)
-    with open(os.path.join(args.model_dir, "feature_columns.json"), "w") as fp:
-        json.dump(features, fp)
-
-
-def model_fn(model_dir) -> dict:
-    """Deserialized and return all the fitted models from the model directory.
-
-    Args:
-        model_dir (str): The directory where the models are stored.
-
-    Returns:
-        dict: A dictionary of the models.
-    """
-
-    # Load ALL the Quantile models from the model directory
-    models = {}
-    for file in os.listdir(model_dir):
-        if file.startswith("q") and file.endswith(".json"):  # The Quantile models
-            # Load the model
-            model_path = os.path.join(model_dir, file)
-            print(f"Loading model: {model_path}")
-            model = xgb.XGBRegressor()
-            model.load_model(model_path)
-
-            # Store the quantile model
-            q_name = os.path.splitext(file)[0]
-            models[q_name] = model
-
-    # Return all the models
-    return models
-
-
-def input_fn(input_data, content_type):
-    """Parse input data and return a DataFrame."""
-    if not input_data:
-        raise ValueError("Empty input data is not supported!")
-
-    # Decode bytes to string if necessary
-    if isinstance(input_data, bytes):
-        input_data = input_data.decode("utf-8")
-
-    if "text/csv" in content_type:
-        return pd.read_csv(StringIO(input_data))
-    elif "application/json" in content_type:
-        return pd.DataFrame(json.loads(input_data))  # Assumes JSON array of records
-    else:
-        raise ValueError(f"{content_type} not supported!")
-
-
-def output_fn(output_df, accept_type):
-    """Supports both CSV and JSON output formats."""
-    if "text/csv" in accept_type:
-        csv_output = output_df.fillna("N/A").to_csv(index=False)  # CSV with N/A for missing values
-        return csv_output, "text/csv"
-    elif "application/json" in accept_type:
-        return output_df.to_json(orient="records"), "application/json"  # JSON array of records (NaNs -> null)
-    else:
-        raise RuntimeError(f"{accept_type} accept type is not supported by this script.")
-
-
-def predict_fn(df, models) -> pd.DataFrame:
-    """Make Predictions with our XGB Quantile Regression Model
-
-    Args:
-        df (pd.DataFrame): The input DataFrame
-        models (dict): The dictionary of models to use for predictions
-
-    Returns:
-        pd.DataFrame: The DataFrame with the predictions added
-    """
-
-    # Grab our feature columns (from training)
-    model_dir = os.environ.get("SM_MODEL_DIR", "/opt/ml/model")
-    with open(os.path.join(model_dir, "feature_columns.json")) as fp:
-        model_features = json.load(fp)
-    print(f"Model Features: {model_features}")
-
-    # We're going match features in a case-insensitive manner, accounting for all the permutations
-    # - Model has a feature list that's any case ("Id", "taCos", "cOunT", "likes_tacos")
-    # - Incoming data has columns that are mixed case ("ID", "Tacos", "Count", "Likes_Tacos")
-    matched_df = match_features_case_insensitive(df, model_features)
-
-    # Predict the features against all the models
-    for name, model in models.items():
-        df[name] = model.predict(matched_df[model_features])
-
-    # Use the median prediction as the main prediction
-    df["prediction"] = df["q_50"]
-
-    # Estimate the standard deviation of the predictions using the interquartile range
-    df["prediction_std"] = (df["q_75"] - df["q_25"]) / 1.35
-
-    # Reorganize the columns so they are in alphabetical order
-    df = df.reindex(sorted(df.columns), axis=1)
-
-    # All done, return the DataFrame
-    return df

workbench/model_scripts/quant_regression/requirements.txt

@@ -1 +0,0 @@
-# Note: In general this file should be empty (as the default inference image has all required libraries)