workbench 0.8.205__py3-none-any.whl → 0.8.213__py3-none-any.whl

workbench/model_scripts/xgb_model/generated_model_script.py

@@ -1,497 +1,434 @@
-# Imports for XGB Model
-import xgboost as xgb
-import awswrangler as wr
-import numpy as np
-
-# Model Performance Scores
-from sklearn.metrics import (
-    mean_absolute_error,
-    median_absolute_error,
-    r2_score,
-    root_mean_squared_error,
-    precision_recall_fscore_support,
-    confusion_matrix,
-)
-from scipy.stats import spearmanr
+# XGBoost Model Template for Workbench
+#
+# This template handles both classification and regression models with:
+# - K-fold cross-validation ensemble training (or single train/val split)
+# - Out-of-fold predictions for validation metrics
+# - Uncertainty quantification for regression models
+# - Sample weights support
+# - Categorical feature handling
+# - Compressed feature decompression
 
-# Classification Encoder
-from sklearn.preprocessing import LabelEncoder
-
-# Scikit Learn Imports
-from sklearn.model_selection import train_test_split
-
-from io import StringIO
-import json
 import argparse
-import joblib
+import json
 import os
+
+import awswrangler as wr
+import joblib
+import numpy as np
 import pandas as pd
-from typing import List, Tuple
+import xgboost as xgb
+from sklearn.model_selection import KFold, StratifiedKFold, train_test_split
+from sklearn.preprocessing import LabelEncoder
 
-# Default Hyperparameters for XGBoost
+from model_script_utils import (
+    check_dataframe,
+    compute_classification_metrics,
+    compute_regression_metrics,
+    convert_categorical_types,
+    decompress_features,
+    expand_proba_column,
+    input_fn,
+    match_features_case_insensitive,
+    output_fn,
+    print_classification_metrics,
+    print_confusion_matrix,
+    print_regression_metrics,
+)
+from uq_harness import (
+    compute_confidence,
+    load_uq_models,
+    predict_intervals,
+    save_uq_models,
+    train_uq_models,
+)
+
+# =============================================================================
+# Default Hyperparameters
+# =============================================================================
 DEFAULT_HYPERPARAMETERS = {
+    # Training parameters
+    "n_folds": 5,  # Number of CV folds (1 = single train/val split)
     # Core tree parameters
-    "n_estimators": 200,  # More trees for better signal capture when we have lots of features
-    "max_depth": 6,  # Medium depth
-    "learning_rate": 0.05,  # Lower rate with more estimators for smoother learning
-
+    "n_estimators": 200,
+    "max_depth": 6,
+    "learning_rate": 0.05,
     # Sampling parameters
-    "subsample": 0.7,  # Moderate row sampling to reduce overfitting
-    "colsample_bytree": 0.6,  # More aggressive feature sampling given lots of features
-    "colsample_bylevel": 0.8,  # Additional feature sampling at each tree level
-
+    "subsample": 0.7,
+    "colsample_bytree": 0.6,
+    "colsample_bylevel": 0.8,
     # Regularization
-    "min_child_weight": 5,  # Higher to prevent overfitting on small groups
-    "gamma": 0.2,  # Moderate pruning - you have real signal so don't over-prune
-    "reg_alpha": 0.5,  # L1 for feature selection (useful with many features)
-    "reg_lambda": 2.0,  # Strong L2 to smooth predictions
-
+    "min_child_weight": 5,
+    "gamma": 0.2,
+    "reg_alpha": 0.5,
+    "reg_lambda": 2.0,
     # Random seed
     "random_state": 42,
 }
 
-# Template Parameters
+# Workbench-specific parameters (not passed to XGBoost)
+WORKBENCH_PARAMS = {"n_folds"}
+
+# Template parameters (filled in by Workbench)
 TEMPLATE_PARAMS = {
-    "model_type": "classifier",
-    "target": "wine_class",
-    "features": ['alcohol', 'malic_acid', 'ash', 'alcalinity_of_ash', 'magnesium', 'total_phenols', 'flavanoids', 'nonflavanoid_phenols', 'proanthocyanins', 'color_intensity', 'hue', 'od280_od315_of_diluted_wines', 'proline'],
+    "model_type": "uq_regressor",
+    "target": "udm_asy_res_efflux_ratio",
+    "features": ['smr_vsa4', 'tpsa', 'numhdonors', 'nhohcount', 'nbase', 'vsa_estate3', 'fr_guanido', 'mollogp', 'peoe_vsa8', 'peoe_vsa1', 'fr_imine', 'vsa_estate2', 'estate_vsa10', 'asphericity', 'xc_3dv', 'smr_vsa3', 'charge_centroid_distance', 'c3sp3', 'nitrogen_span', 'estate_vsa2', 'minpartialcharge', 'hba_hbd_ratio', 'slogp_vsa1', 'axp_7d', 'nocount', 'vsa_estate4', 'vsa_estate6', 'estate_vsa4', 'xc_4dv', 'xc_4d', 'num_s_centers', 'vsa_estate9', 'chi2v', 'axp_5d', 'mi', 'mse', 'bcut2d_mrhi', 'smr_vsa6', 'hallkieralpha', 'balabanj', 'amphiphilic_moment', 'type_ii_pattern_count', 'minabsestateindex', 'bcut2d_mwlow', 'axp_0dv', 'slogp_vsa5', 'axp_2d', 'axp_1dv', 'xch_5d', 'peoe_vsa10', 'molecular_asymmetry', 'kappa3', 'estate_vsa3', 'sse', 'bcut2d_logphi', 'fr_imidazole', 'molecular_volume_3d', 'bertzct', 'maxestateindex', 'aromatic_interaction_score', 'axp_3d', 'radius_of_gyration', 'vsa_estate7', 'si', 'axp_5dv', 'molecular_axis_length', 'estate_vsa6', 'fpdensitymorgan1', 'axp_6d', 'estate_vsa9', 'fpdensitymorgan2', 'xp_0dv', 'xp_6dv', 'molmr', 'qed', 'estate_vsa8', 'peoe_vsa9', 'xch_6dv', 'xp_7d', 'slogp_vsa2', 'xp_5dv', 'bcut2d_chghi', 'xch_6d', 'chi0n', 'slogp_vsa3', 'chi1v', 'chi3v', 'bcut2d_chglo', 'axp_1d', 'mp', 'num_defined_stereocenters', 'xp_3dv', 'bcut2d_mrlow', 'fr_al_oh', 'peoe_vsa7', 'chi2n', 'axp_6dv', 'axp_2dv', 'chi4n', 'xc_3d', 'axp_7dv', 'vsa_estate8', 'xch_7d', 'maxpartialcharge', 'chi1n', 'peoe_vsa2', 'axp_3dv', 'bcut2d_logplow', 'mv', 'xpc_5dv', 'kappa2', 'vsa_estate5', 'xp_5d', 'mm', 'maxabspartialcharge', 'axp_4dv', 'maxabsestateindex', 'axp_4d', 'xch_4dv', 'xp_2dv', 'heavyatommolwt', 'numatomstereocenters', 'xp_7dv', 'numsaturatedheterocycles', 'xp_3d', 'kappa1', 'mz', 'axp_0d', 'chi1', 'xch_4d', 'smr_vsa1', 'xp_2d', 'estate_vsa5', 'phi', 'fr_ether', 'xc_5d', 'c1sp3', 'estate_vsa7', 'estate_vsa1', 'vsa_estate1', 'slogp_vsa4', 'avgipc', 'smr_vsa10', 'numvalenceelectrons', 'xc_5dv', 'peoe_vsa12', 'peoe_vsa6', 'xpc_5d', 'xpc_6d', 'minestateindex', 'chi3n', 'smr_vsa5', 'xp_4d', 'numheteroatoms', 'fpdensitymorgan3', 'xpc_4d', 'sps', 'xp_1d', 'sv', 'fr_ar_n', 'slogp_vsa10', 'c2sp3', 'xpc_4dv', 'chi0v', 'xpc_6dv', 'xp_1dv', 'vsa_estate10', 'sare', 'c2sp2', 'mpe', 'xch_7dv', 'chi4v', 'type_i_pattern_count', 'sp', 'slogp_vsa8', 'amide_count', 'num_stereocenters', 'num_r_centers', 'tertiary_amine_count', 'spe', 'xp_4dv', 'numsaturatedrings', 'mare', 'numhacceptors', 'chi0', 'fractioncsp3', 'fr_nh0', 'xch_5dv', 'fr_aniline', 'smr_vsa7', 'labuteasa', 'c3sp2', 'xp_0d', 'xp_6d', 'peoe_vsa11', 'fr_ar_nh', 'molwt', 'intramolecular_hbond_potential', 'peoe_vsa3', 'fr_nhpyrrole', 'numaliphaticrings', 'hybratio', 'smr_vsa9', 'peoe_vsa13', 'bcut2d_mwhi', 'c1sp2', 'slogp_vsa11', 'numrotatablebonds', 'numaliphaticcarbocycles', 'slogp_vsa6', 'peoe_vsa4', 'numunspecifiedatomstereocenters', 'xc_6d', 'xc_6dv', 'num_unspecified_stereocenters', 'sz', 'minabspartialcharge', 'fcsp3', 'c1sp1', 'fr_piperzine', 'numaliphaticheterocycles', 'numamidebonds', 'fr_benzene', 'numaromaticheterocycles', 'sm', 'fr_priamide', 'fr_piperdine', 'fr_methoxy', 'c4sp3', 'fr_c_o_nocoo', 'exactmolwt', 'stereo_complexity', 'fr_hoccn', 'numaromaticcarbocycles', 'fr_nh2', 'numheterocycles', 'fr_morpholine', 'fr_ketone', 'fr_nh1', 'frac_defined_stereo', 'fr_aryl_methyl', 'fr_alkyl_halide', 'fr_phenol', 'fr_al_oh_notert', 'fr_ar_oh', 'fr_pyridine', 'fr_amide', 'slogp_vsa7', 'fr_halogen', 'numsaturatedcarbocycles', 'slogp_vsa12', 'fr_ndealkylation1', 'xch_3d', 'fr_bicyclic', 'naromatom', 'narombond'],
+    "id_column": "udm_mol_bat_id",
     "compressed_features": [],
-    "model_metrics_s3_path": "s3://sandbox-sageworks-artifacts/models/wine-classification/training",
-    "train_all_data": False,
-    "hyperparameters": {},
+    "model_metrics_s3_path": "s3://ideaya-sageworks-bucket/models/caco2-er-reg-test-log/training",
+    "hyperparameters": {'target_transform': 'log'},
 }
 
 
-# Function to check if dataframe is empty
-def check_dataframe(df: pd.DataFrame, df_name: str) -> None:
-    """
-    Check if the provided dataframe is empty and raise an exception if it is.
-
-    Args:
-        df (pd.DataFrame): DataFrame to check
-        df_name (str): Name of the DataFrame
-    """
-    if df.empty:
-        msg = f"*** The training data {df_name} has 0 rows! ***STOPPING***"
-        print(msg)
-        raise ValueError(msg)
-
-
-def expand_proba_column(df: pd.DataFrame, class_labels: List[str]) -> pd.DataFrame:
-    """
-    Expands a column in a DataFrame containing a list of probabilities into separate columns.
-
-    Args:
-        df (pd.DataFrame): DataFrame containing a "pred_proba" column
-        class_labels (List[str]): List of class labels
-
-    Returns:
-        pd.DataFrame: DataFrame with the "pred_proba" expanded into separate columns
-    """
-
-    # Sanity check
-    proba_column = "pred_proba"
-    if proba_column not in df.columns:
-        raise ValueError('DataFrame does not contain a "pred_proba" column')
-
-    # Construct new column names with '_proba' suffix
-    proba_splits = [f"{label}_proba" for label in class_labels]
-
-    # Expand the proba_column into separate columns for each probability
-    proba_df = pd.DataFrame(df[proba_column].tolist(), columns=proba_splits)
-
-    # Drop any proba columns and reset the index in prep for the concat
-    df = df.drop(columns=[proba_column] + proba_splits, errors="ignore")
-    df = df.reset_index(drop=True)
-
-    # Concatenate the new columns with the original DataFrame
-    df = pd.concat([df, proba_df], axis=1)
-    print(df)
-    return df
-
-
-def match_features_case_insensitive(df: pd.DataFrame, model_features: list) -> pd.DataFrame:
-    """
-    Matches and renames DataFrame columns to match model feature names (case-insensitive).
-    Prioritizes exact matches, then case-insensitive matches.
-
-    Raises ValueError if any model features cannot be matched.
-    """
-    df_columns_lower = {col.lower(): col for col in df.columns}
-    rename_dict = {}
-    missing = []
-    for feature in model_features:
-        if feature in df.columns:
-            continue  # Exact match
-        elif feature.lower() in df_columns_lower:
-            rename_dict[df_columns_lower[feature.lower()]] = feature
-        else:
-            missing.append(feature)
-
-    if missing:
-        raise ValueError(f"Features not found: {missing}")
-
-    # Rename the DataFrame columns to match the model features
-    return df.rename(columns=rename_dict)
-
-
-def convert_categorical_types(df: pd.DataFrame, features: list, category_mappings={}) -> tuple:
-    """
-    Converts appropriate columns to categorical type with consistent mappings.
-
-    Args:
-        df (pd.DataFrame): The DataFrame to process.
-        features (list): List of feature names to consider for conversion.
-        category_mappings (dict, optional): Existing category mappings. If empty dict, we're in
-                                            training mode. If populated, we're in inference mode.
-
-    Returns:
-        tuple: (processed DataFrame, category mappings dictionary)
-    """
-    # Training mode
-    if category_mappings == {}:
-        for col in df.select_dtypes(include=["object", "string"]):
-            if col in features and df[col].nunique() < 20:
-                print(f"Training mode: Converting {col} to category")
-                df[col] = df[col].astype("category")
-                category_mappings[col] = df[col].cat.categories.tolist()  # Store category mappings
-
-    # Inference mode
-    else:
-        for col, categories in category_mappings.items():
-            if col in df.columns:
-                print(f"Inference mode: Applying categorical mapping for {col}")
-                df[col] = pd.Categorical(df[col], categories=categories)  # Apply consistent categorical mapping
-
-    return df, category_mappings
-
-
-def decompress_features(
-    df: pd.DataFrame, features: List[str], compressed_features: List[str]
-) -> Tuple[pd.DataFrame, List[str]]:
-    """Prepare features for the model by decompressing bitstring features
-
-    Args:
-        df (pd.DataFrame): The features DataFrame
-        features (List[str]): Full list of feature names
-        compressed_features (List[str]): List of feature names to decompress (bitstrings)
-
-    Returns:
-        pd.DataFrame: DataFrame with the decompressed features
-        List[str]: Updated list of feature names after decompression
-
-    Raises:
-        ValueError: If any missing values are found in the specified features
-    """
-
-    # Check for any missing values in the required features
-    missing_counts = df[features].isna().sum()
-    if missing_counts.any():
-        missing_features = missing_counts[missing_counts > 0]
-        print(
-            f"WARNING: Found missing values in features: {missing_features.to_dict()}. "
-            "WARNING: You might want to remove/replace all NaN values before processing."
-        )
-
-    # Decompress the specified compressed features
-    decompressed_features = features.copy()
-    for feature in compressed_features:
-        if (feature not in df.columns) or (feature not in features):
-            print(f"Feature '{feature}' not in the features list, skipping decompression.")
-            continue
-
-        # Remove the feature from the list of features to avoid duplication
-        decompressed_features.remove(feature)
-
-        # Handle all compressed features as bitstrings
-        bit_matrix = np.array([list(bitstring) for bitstring in df[feature]], dtype=np.uint8)
-        prefix = feature[:3]
-
-        # Create all new columns at once - avoids fragmentation
-        new_col_names = [f"{prefix}_{i}" for i in range(bit_matrix.shape[1])]
-        new_df = pd.DataFrame(bit_matrix, columns=new_col_names, index=df.index)
-
-        # Add to features list
-        decompressed_features.extend(new_col_names)
-
-        # Drop original column and concatenate new ones
-        df = df.drop(columns=[feature])
-        df = pd.concat([df, new_df], axis=1)
-
-    return df, decompressed_features
-
-
+# =============================================================================
+# Training
+# =============================================================================
 if __name__ == "__main__":
-    """The main function is for training the XGBoost model"""
+    # -------------------------------------------------------------------------
+    # Setup: Parse arguments and load data
+    # -------------------------------------------------------------------------
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-dir", type=str, default=os.environ.get("SM_MODEL_DIR", "/opt/ml/model"))
+    parser.add_argument("--train", type=str, default=os.environ.get("SM_CHANNEL_TRAIN", "/opt/ml/input/data/train"))
+    parser.add_argument("--output-data-dir", type=str, default=os.environ.get("SM_OUTPUT_DATA_DIR", "/opt/ml/output/data"))
+    args = parser.parse_args()
 
-    # Harness Template Parameters
+    # Extract template parameters
     target = TEMPLATE_PARAMS["target"]
     features = TEMPLATE_PARAMS["features"]
     orig_features = features.copy()
+    id_column = TEMPLATE_PARAMS["id_column"]
     compressed_features = TEMPLATE_PARAMS["compressed_features"]
     model_type = TEMPLATE_PARAMS["model_type"]
     model_metrics_s3_path = TEMPLATE_PARAMS["model_metrics_s3_path"]
-    train_all_data = TEMPLATE_PARAMS["train_all_data"]
-    hyperparameters = {**DEFAULT_HYPERPARAMETERS, **TEMPLATE_PARAMS["hyperparameters"]}
-    validation_split = 0.2
-
-    # Script arguments for input/output directories
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--model-dir", type=str, default=os.environ.get("SM_MODEL_DIR", "/opt/ml/model"))
-    parser.add_argument("--train", type=str, default=os.environ.get("SM_CHANNEL_TRAIN", "/opt/ml/input/data/train"))
-    parser.add_argument(
-        "--output-data-dir", type=str, default=os.environ.get("SM_OUTPUT_DATA_DIR", "/opt/ml/output/data")
-    )
-    args = parser.parse_args()
+    hyperparameters = {**DEFAULT_HYPERPARAMETERS, **(TEMPLATE_PARAMS["hyperparameters"] or {})}
 
-    # Read the training data into DataFrames
-    training_files = [os.path.join(args.train, file) for file in os.listdir(args.train) if file.endswith(".csv")]
+    # Load training data
+    training_files = [os.path.join(args.train, f) for f in os.listdir(args.train) if f.endswith(".csv")]
     print(f"Training Files: {training_files}")
-
-    # Combine files and read them all into a single pandas dataframe
-    all_df = pd.concat([pd.read_csv(file, engine="python") for file in training_files])
-
-    # Check if the dataframe is empty
+    all_df = pd.concat([pd.read_csv(f, engine="python") for f in training_files])
     check_dataframe(all_df, "training_df")
 
-    # Features/Target output
     print(f"Target: {target}")
-    print(f"Features: {str(features)}")
+    print(f"Features: {features}")
+    print(f"Hyperparameters: {hyperparameters}")
 
-    # Convert any features that might be categorical to 'category' type
+    # -------------------------------------------------------------------------
+    # Preprocessing: Categorical features and decompression
+    # -------------------------------------------------------------------------
     all_df, category_mappings = convert_categorical_types(all_df, features)
 
-    # If we have compressed features, decompress them
     if compressed_features:
-        print(f"Decompressing features {compressed_features}...")
+        print(f"Decompressing features: {compressed_features}")
         all_df, features = decompress_features(all_df, features, compressed_features)
 
-    # Do we want to train on all the data?
-    if train_all_data:
-        print("Training on ALL of the data")
-        df_train = all_df.copy()
-        df_val = all_df.copy()
-
-    # Does the dataframe have a training column?
-    elif "training" in all_df.columns:
-        print("Found training column, splitting data based on training column")
-        df_train = all_df[all_df["training"]]
-        df_val = all_df[~all_df["training"]]
+    # -------------------------------------------------------------------------
+    # Classification setup: Encode target labels
+    # -------------------------------------------------------------------------
+    label_encoder = None
+    if model_type == "classifier":
+        label_encoder = LabelEncoder()
+        all_df[target] = label_encoder.fit_transform(all_df[target])
+        print(f"Class labels: {label_encoder.classes_.tolist()}")
+
+    # -------------------------------------------------------------------------
+    # Cross-validation setup
+    # -------------------------------------------------------------------------
+    n_folds = hyperparameters["n_folds"]
+    xgb_params = {k: v for k, v in hyperparameters.items() if k not in WORKBENCH_PARAMS}
+    print(f"XGBoost params: {xgb_params}")
+
+    if n_folds == 1:
+        # Single train/val split
+        if "training" in all_df.columns:
+            print("Using 'training' column for train/val split")
+            train_idx = np.where(all_df["training"])[0]
+            val_idx = np.where(~all_df["training"])[0]
+        else:
+            print("WARNING: No 'training' column found, using random 80/20 split")
+            indices = np.arange(len(all_df))
+            train_idx, val_idx = train_test_split(indices, test_size=0.2, random_state=42)
+        folds = [(train_idx, val_idx)]
     else:
-        # Just do a random training Split
-        print("WARNING: No training column found, splitting data with random state=42")
-        df_train, df_val = train_test_split(all_df, test_size=validation_split, random_state=42)
-    print(f"FIT/TRAIN: {df_train.shape}")
-    print(f"VALIDATION: {df_val.shape}")
+        # K-fold cross-validation
+        if model_type == "classifier":
+            kfold = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=42)
+            folds = list(kfold.split(all_df, all_df[target]))
+        else:
+            kfold = KFold(n_splits=n_folds, shuffle=True, random_state=42)
+            folds = list(kfold.split(all_df))
 
-    # Use any hyperparameters to set up both the trainer and model configurations
-    print(f"Hyperparameters: {hyperparameters}")
+    print(f"Training {'single model' if n_folds == 1 else f'{n_folds}-fold ensemble'}...")
 
-    # Now spin up our XGB Model
+    # -------------------------------------------------------------------------
+    # Training loop
+    # -------------------------------------------------------------------------
+    # Initialize out-of-fold storage
+    oof_predictions = np.full(len(all_df), np.nan, dtype=np.float64)
     if model_type == "classifier":
-        xgb_model = xgb.XGBClassifier(enable_categorical=True, **hyperparameters)
-
-        # Encode the target column
-        label_encoder = LabelEncoder()
-        df_train[target] = label_encoder.fit_transform(df_train[target])
-        df_val[target] = label_encoder.transform(df_val[target])
-
+        num_classes = len(label_encoder.classes_)
+        oof_proba = np.full((len(all_df), num_classes), np.nan, dtype=np.float64)
     else:
-        xgb_model = xgb.XGBRegressor(enable_categorical=True, **hyperparameters)
-        label_encoder = None  # We don't need this for regression
-
-    # Grab our Features, Target and Train the Model
-    y_train = df_train[target]
-    X_train = df_train[features]
-    xgb_model.fit(X_train, y_train)
-
-    # Make Predictions on the Validation Set
-    print(f"Making Predictions on Validation Set...")
-    y_validate = df_val[target]
-    X_validate = df_val[features]
-    preds = xgb_model.predict(X_validate)
+        oof_proba = None
+
+    # Check for sample weights
+    has_sample_weights = "sample_weight" in all_df.columns
+    if has_sample_weights:
+        sw = all_df["sample_weight"]
+        print(f"Using sample weights: min={sw.min():.2f}, max={sw.max():.2f}, mean={sw.mean():.2f}")
+
+    # Train ensemble
+    ensemble_models = []
+    for fold_idx, (train_idx, val_idx) in enumerate(folds):
+        print(f"\n{'='*50}")
+        print(f"Fold {fold_idx + 1}/{len(folds)} - Train: {len(train_idx)}, Val: {len(val_idx)}")
+        print(f"{'='*50}")
+
+        # Prepare fold data
+        X_train = all_df.iloc[train_idx][features]
+        y_train = all_df.iloc[train_idx][target]
+        X_val = all_df.iloc[val_idx][features]
+        sample_weights = all_df.iloc[train_idx]["sample_weight"] if has_sample_weights else None
+
+        # Create model with fold-specific random state for diversity
+        fold_params = {**xgb_params, "random_state": xgb_params.get("random_state", 42) + fold_idx}
+        if model_type == "classifier":
+            model = xgb.XGBClassifier(enable_categorical=True, **fold_params)
+        else:
+            model = xgb.XGBRegressor(enable_categorical=True, **fold_params)
+
+        # Train
+        model.fit(X_train, y_train, sample_weight=sample_weights)
+        ensemble_models.append(model)
+
+        # Out-of-fold predictions
+        oof_predictions[val_idx] = model.predict(X_val)
+        if model_type == "classifier":
+            oof_proba[val_idx] = model.predict_proba(X_val)
+
+    print(f"\nTraining complete! Trained {len(ensemble_models)} model(s).")
+
+    # -------------------------------------------------------------------------
+    # Prepare validation results
+    # -------------------------------------------------------------------------
+    if n_folds == 1:
+        # Single fold: only validation rows
+        val_mask = ~np.isnan(oof_predictions)
+        df_val = all_df[val_mask].copy()
+        predictions = oof_predictions[val_mask]
+        if oof_proba is not None:
+            oof_proba = oof_proba[val_mask]
+    else:
+        # K-fold: all rows have out-of-fold predictions
+        df_val = all_df.copy()
+        predictions = oof_predictions
+
+    # Decode labels for classification
     if model_type == "classifier":
-        # Also get the probabilities for each class
-        print("Processing Probabilities...")
-        probs = xgb_model.predict_proba(X_validate)
-        df_val["pred_proba"] = [p.tolist() for p in probs]
-
-        # Expand the pred_proba column into separate columns for each class
-        print(df_val.columns)
-        df_val = expand_proba_column(df_val, label_encoder.classes_)
-        print(df_val.columns)
-
-        # Decode the target and prediction labels
-        y_validate = label_encoder.inverse_transform(y_validate)
-        preds = label_encoder.inverse_transform(preds)
-
-    # Save predictions to S3 (just the target, prediction, and '_proba' columns)
-    df_val["prediction"] = preds
-    output_columns = [target, "prediction"]
-    output_columns += [col for col in df_val.columns if col.endswith("_proba")]
-    wr.s3.to_csv(
-        df_val[output_columns],
-        path=f"{model_metrics_s3_path}/validation_predictions.csv",
-        index=False,
-    )
-
-    # Report Performance Metrics
+        df_val[target] = label_encoder.inverse_transform(df_val[target].astype(int))
+        df_val["prediction"] = label_encoder.inverse_transform(predictions.astype(int))
+        if oof_proba is not None:
+            df_val["pred_proba"] = [p.tolist() for p in oof_proba]
+            df_val = expand_proba_column(df_val, label_encoder.classes_)
+    else:
+        df_val["prediction"] = predictions
+
+    # -------------------------------------------------------------------------
+    # Compute and print metrics
+    # -------------------------------------------------------------------------
+    y_true = df_val[target].values
+    y_pred = df_val["prediction"].values
+
     if model_type == "classifier":
-        # Get the label names and their integer mapping
         label_names = label_encoder.classes_
-
-        # Calculate various model performance metrics
-        scores = precision_recall_fscore_support(y_validate, preds, average=None, labels=label_names)
-
-        # Put the scores into a dataframe
-        score_df = pd.DataFrame(
-            {
-                target: label_names,
-                "precision": scores[0],
-                "recall": scores[1],
-                "f1": scores[2],
-                "support": scores[3],
-            }
+        score_df = compute_classification_metrics(y_true, y_pred, label_names, target)
+        print_classification_metrics(score_df, target, label_names)
+        print_confusion_matrix(y_true, y_pred, label_names)
+    else:
+        metrics = compute_regression_metrics(y_true, y_pred)
+        print_regression_metrics(metrics)
+
+        # Compute ensemble prediction_std
+        if n_folds > 1:
+            all_preds = np.stack([m.predict(all_df[features]) for m in ensemble_models])
+            df_val["prediction_std"] = np.std(all_preds, axis=0)
+            print(f"Ensemble std - mean: {df_val['prediction_std'].mean():.4f}, max: {df_val['prediction_std'].max():.4f}")
+        else:
+            df_val["prediction_std"] = 0.0
+
+        # Train UQ models for uncertainty quantification
+        print("\n" + "=" * 50)
+        print("Training UQ Models")
+        print("=" * 50)
+        uq_models, uq_metadata = train_uq_models(
+            all_df[features], all_df[target], df_val[features], y_true
         )
+        df_val = predict_intervals(df_val, df_val[features], uq_models, uq_metadata)
+        df_val = compute_confidence(df_val, uq_metadata["median_interval_width"])
 
-        # We need to get creative with the Classification Metrics
-        metrics = ["precision", "recall", "f1", "support"]
-        for t in label_names:
-            for m in metrics:
-                value = score_df.loc[score_df[target] == t, m].iloc[0]
-                print(f"Metrics:{t}:{m} {value}")
+    # -------------------------------------------------------------------------
+    # Save validation predictions to S3
+    # -------------------------------------------------------------------------
+    output_columns = []
+    if id_column in df_val.columns:
+        output_columns.append(id_column)
+    output_columns += [target, "prediction"]
 
-        # Compute and output the confusion matrix
-        conf_mtx = confusion_matrix(y_validate, preds, labels=label_names)
-        for i, row_name in enumerate(label_names):
-            for j, col_name in enumerate(label_names):
-                value = conf_mtx[i, j]
-                print(f"ConfusionMatrix:{row_name}:{col_name} {value}")
+    if model_type != "classifier":
+        output_columns.append("prediction_std")
+        output_columns += [c for c in df_val.columns if c.startswith("q_") or c == "confidence"]
 
-    else:
-        # Calculate various model performance metrics (regression)
-        rmse = root_mean_squared_error(y_validate, preds)
-        mae = mean_absolute_error(y_validate, preds)
-        medae = median_absolute_error(y_validate, preds)
-        r2 = r2_score(y_validate, preds)
-        spearman_corr = spearmanr(y_validate, preds).correlation
-        support = len(df_val)
-        print(f"rmse: {rmse:.3f}")
-        print(f"mae: {mae:.3f}")
-        print(f"medae: {medae:.3f}")
-        print(f"r2: {r2:.3f}")
-        print(f"spearmanr: {spearman_corr:.3f}")
-        print(f"support: {support}")
-
-    # Now save the model to the standard place/name
-    joblib.dump(xgb_model, os.path.join(args.model_dir, "xgb_model.joblib"))
-
-    # Save the label encoder if we have one
-    if label_encoder:
-        joblib.dump(label_encoder, os.path.join(args.model_dir, "label_encoder.joblib"))
+    output_columns += [c for c in df_val.columns if c.endswith("_proba")]
 
-    # Save the features (this will validate input during predictions)
-    with open(os.path.join(args.model_dir, "feature_columns.json"), "w") as fp:
-        json.dump(orig_features, fp)  # We save the original features, not the decompressed ones
+    wr.s3.to_csv(df_val[output_columns], f"{model_metrics_s3_path}/validation_predictions.csv", index=False)
 
-    # Save the category mappings
-    with open(os.path.join(args.model_dir, "category_mappings.json"), "w") as fp:
-        json.dump(category_mappings, fp)
+    # -------------------------------------------------------------------------
+    # Save model artifacts
+    # -------------------------------------------------------------------------
+    # Ensemble models
+    for idx, ens_model in enumerate(ensemble_models):
+        joblib.dump(ens_model, os.path.join(args.model_dir, f"xgb_model_{idx}.joblib"))
+    print(f"Saved {len(ensemble_models)} XGBoost model(s)")
 
+    # Metadata files
+    with open(os.path.join(args.model_dir, "ensemble_metadata.json"), "w") as f:
+        json.dump({"n_ensemble": len(ensemble_models), "n_folds": n_folds}, f)
 
-def model_fn(model_dir):
-    """Deserialize and return fitted XGBoost model"""
-    model_path = os.path.join(model_dir, "xgb_model.joblib")
-    model = joblib.load(model_path)
-    return model
+    with open(os.path.join(args.model_dir, "feature_columns.json"), "w") as f:
+        json.dump(orig_features, f)
 
+    with open(os.path.join(args.model_dir, "category_mappings.json"), "w") as f:
+        json.dump(category_mappings, f)
 
-def input_fn(input_data, content_type):
-    """Parse input data and return a DataFrame."""
-    if not input_data:
-        raise ValueError("Empty input data is not supported!")
+    with open(os.path.join(args.model_dir, "hyperparameters.json"), "w") as f:
+        json.dump(hyperparameters, f, indent=2)
 
-    # Decode bytes to string if necessary
-    if isinstance(input_data, bytes):
-        input_data = input_data.decode("utf-8")
+    if label_encoder:
+        joblib.dump(label_encoder, os.path.join(args.model_dir, "label_encoder.joblib"))
 
-    if "text/csv" in content_type:
-        return pd.read_csv(StringIO(input_data))
-    elif "application/json" in content_type:
-        return pd.DataFrame(json.loads(input_data))  # Assumes JSON array of records
-    else:
-        raise ValueError(f"{content_type} not supported!")
+    if model_type != "classifier":
+        save_uq_models(uq_models, uq_metadata, args.model_dir)
+
+    print(f"\nModel training complete! Artifacts saved to {args.model_dir}")
 
 
-def output_fn(output_df, accept_type):
-    """Supports both CSV and JSON output formats."""
-    if "text/csv" in accept_type:
-        csv_output = output_df.fillna("N/A").to_csv(index=False)  # CSV with N/A for missing values
-        return csv_output, "text/csv"
-    elif "application/json" in accept_type:
-        return output_df.to_json(orient="records"), "application/json"  # JSON array of records (NaNs -> null)
+# =============================================================================
+# Model Loading (for SageMaker inference)
+# =============================================================================
+def model_fn(model_dir: str) -> dict:
+    """Load XGBoost ensemble and associated artifacts.
+
+    Args:
+        model_dir: Directory containing model artifacts
+
+    Returns:
+        Dictionary with ensemble_models, label_encoder, category_mappings, uq_models, etc.
+    """
+    # Load ensemble metadata
+    metadata_path = os.path.join(model_dir, "ensemble_metadata.json")
+    if os.path.exists(metadata_path):
+        with open(metadata_path) as f:
+            metadata = json.load(f)
+        n_ensemble = metadata["n_ensemble"]
     else:
-        raise RuntimeError(f"{accept_type} accept type is not supported by this script.")
+        n_ensemble = 1  # Legacy single model
 
+    # Load ensemble models
+    ensemble_models = []
+    for i in range(n_ensemble):
+        model_path = os.path.join(model_dir, f"xgb_model_{i}.joblib")
+        if not os.path.exists(model_path):
+            model_path = os.path.join(model_dir, "xgb_model.joblib")  # Legacy fallback
+        ensemble_models.append(joblib.load(model_path))
 
-def predict_fn(df, model) -> pd.DataFrame:
-    """Make Predictions with our XGB Model
+    # Load label encoder (classifier only)
+    label_encoder = None
+    encoder_path = os.path.join(model_dir, "label_encoder.joblib")
+    if os.path.exists(encoder_path):
+        label_encoder = joblib.load(encoder_path)
+
+    # Load category mappings
+    category_mappings = {}
+    category_path = os.path.join(model_dir, "category_mappings.json")
+    if os.path.exists(category_path):
+        with open(category_path) as f:
+            category_mappings = json.load(f)
+
+    # Load UQ models (regression only)
+    uq_models, uq_metadata = None, None
+    uq_path = os.path.join(model_dir, "uq_metadata.json")
+    if os.path.exists(uq_path):
+        uq_models, uq_metadata = load_uq_models(model_dir)
+
+    return {
+        "ensemble_models": ensemble_models,
+        "n_ensemble": n_ensemble,
+        "label_encoder": label_encoder,
+        "category_mappings": category_mappings,
+        "uq_models": uq_models,
+        "uq_metadata": uq_metadata,
+    }
+
+
+# =============================================================================
+# Inference (for SageMaker inference)
+# =============================================================================
+def predict_fn(df: pd.DataFrame, models: dict) -> pd.DataFrame:
+    """Make predictions with XGBoost ensemble.
 
     Args:
-        df (pd.DataFrame): The input DataFrame
-        model: The model use for predictions
+        df: Input DataFrame with features
+        models: Dictionary from model_fn containing ensemble and metadata
 
     Returns:
-        pd.DataFrame: The DataFrame with the predictions added
+        DataFrame with predictions added
     """
-    compressed_features = TEMPLATE_PARAMS["compressed_features"]
-
-    # Grab our feature columns (from training)
+    # Load feature columns
     model_dir = os.environ.get("SM_MODEL_DIR", "/opt/ml/model")
-    with open(os.path.join(model_dir, "feature_columns.json")) as fp:
-        features = json.load(fp)
+    with open(os.path.join(model_dir, "feature_columns.json")) as f:
+        features = json.load(f)
     print(f"Model Features: {features}")
 
-    # Load the category mappings (from training)
-    with open(os.path.join(model_dir, "category_mappings.json")) as fp:
-        category_mappings = json.load(fp)
-
-    # Load our Label Encoder if we have one
-    label_encoder = None
-    if os.path.exists(os.path.join(model_dir, "label_encoder.joblib")):
-        label_encoder = joblib.load(os.path.join(model_dir, "label_encoder.joblib"))
+    # Extract model components
+    ensemble_models = models["ensemble_models"]
+    label_encoder = models.get("label_encoder")
+    category_mappings = models.get("category_mappings", {})
+    uq_models = models.get("uq_models")
+    uq_metadata = models.get("uq_metadata")
+    compressed_features = TEMPLATE_PARAMS["compressed_features"]
 
-    # We're going match features in a case-insensitive manner, accounting for all the permutations
-    # - Model has a feature list that's any case ("Id", "taCos", "cOunT", "likes_tacos")
-    # - Incoming data has columns that are mixed case ("ID", "Tacos", "Count", "Likes_Tacos")
+    # Prepare features
     matched_df = match_features_case_insensitive(df, features)
-
-    # Detect categorical types in the incoming DataFrame
     matched_df, _ = convert_categorical_types(matched_df, features, category_mappings)
 
-    # If we have compressed features, decompress them
     if compressed_features:
         print("Decompressing features for prediction...")
         matched_df, features = decompress_features(matched_df, features, compressed_features)
 
-    # Predict the features against our XGB Model
     X = matched_df[features]
-    predictions = model.predict(X)
 
-    # If we have a label encoder, decode the predictions
-    if label_encoder:
-        predictions = label_encoder.inverse_transform(predictions)
-
-    # Set the predictions on the DataFrame
-    df["prediction"] = predictions
+    # Collect ensemble predictions
+    all_preds = [m.predict(X) for m in ensemble_models]
+    ensemble_preds = np.stack(all_preds, axis=0)
 
-    # Does our model have a 'predict_proba' method? If so we will call it and add the results to the DataFrame
-    if getattr(model, "predict_proba", None):
-        probs = model.predict_proba(matched_df[features])
-        df["pred_proba"] = [p.tolist() for p in probs]
+    if label_encoder is not None:
+        # Classification: average probabilities, then argmax
+        all_probs = [m.predict_proba(X) for m in ensemble_models]
+        avg_probs = np.mean(np.stack(all_probs, axis=0), axis=0)
+        class_preds = np.argmax(avg_probs, axis=1)
 
-        # Expand the pred_proba column into separate columns for each class
+        df["prediction"] = label_encoder.inverse_transform(class_preds)
+        df["pred_proba"] = [p.tolist() for p in avg_probs]
         df = expand_proba_column(df, label_encoder.classes_)
+    else:
+        # Regression: average predictions
+        df["prediction"] = np.mean(ensemble_preds, axis=0)
+        df["prediction_std"] = np.std(ensemble_preds, axis=0)
+
+        # Add UQ intervals if available
+        if uq_models and uq_metadata:
+            df = predict_intervals(df, X, uq_models, uq_metadata)
+            df = compute_confidence(df, uq_metadata["median_interval_width"], "q_10", "q_90")
 
-    # All done, return the DataFrame with new columns for the predictions
+    print(f"Inference complete: {len(df)} predictions, {len(ensemble_models)} ensemble members")
     return df