workbench 0.8.228__tar.gz → 0.8.230__tar.gz

{workbench-0.8.228 → workbench-0.8.230}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: workbench
-Version: 0.8.228
+Version: 0.8.230
 Summary: Workbench: A Dashboard and Python API for creating and deploying AWS SageMaker Model Pipelines
 Author-email: SuperCowPowers LLC <support@supercowpowers.com>
 License: MIT License
@@ -60,7 +60,7 @@ Requires-Dist: plotly>=6.0.0; extra == "ui"
 Requires-Dist: dash>=3.0.0; extra == "ui"
 Requires-Dist: dash-bootstrap-components>=1.6.0; extra == "ui"
 Requires-Dist: dash-bootstrap-templates>=1.3.0; extra == "ui"
-Requires-Dist: dash_ag_grid; extra == "ui"
+Requires-Dist: dash_ag_grid<=32.3.2; extra == "ui"
 Requires-Dist: tabulate>=0.9.0; extra == "ui"
 Requires-Dist: matplotlib>=3.9.2; extra == "ui"
 Provides-Extra: dev

{workbench-0.8.228 → workbench-0.8.230}/applications/aws_dashboard/Dockerfile

@@ -1,4 +1,16 @@
-# Base image: python:3.12.5-bookworm
+# Define workbench version once at the top
+ARG WORKBENCH_VERSION=0.8.228
+
+# Stage 1: Extract dependencies from workbench[ui]
+FROM python:3.12.5-bookworm AS deps-extractor
+ARG WORKBENCH_VERSION
+RUN pip install --no-cache-dir pip-tools && \
+    pip download --no-deps "workbench[ui]==${WORKBENCH_VERSION}" -d /tmp/pkg && \
+    cd /tmp/pkg && unzip -q *.whl -d /tmp/extracted && \
+    cat /tmp/extracted/*.dist-info/METADATA | grep "^Requires-Dist:" | \
+    sed 's/Requires-Dist: //' | sed 's/;.*//' | sed 's/ (.*//' > /tmp/deps.txt
+
+# Stage 2: Build the final image
 FROM python:3.12.5-bookworm AS base
 WORKDIR /app
 EXPOSE 8000
@@ -6,9 +18,6 @@ EXPOSE 8000
 # Set PYTHONHASHSEED to ensure consistent hashing across processes
 ENV PYTHONHASHSEED=0
 
-# Define workbench version
-ARG WORKBENCH_VERSION=0.8.227
-
 # Install system packages and gunicorn in one layer
 RUN apt-get update && apt-get install -y --no-install-recommends \
     nginx supervisor libnghttp2-dev && \
@@ -16,13 +25,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     rm -rf /var/lib/apt/lists/* && \
     pip install --no-cache-dir gunicorn
 
-# Dependency Layer: Install all dependencies of workbench[ui]
-RUN pip install --no-cache-dir "workbench[ui]==${WORKBENCH_VERSION}" && \
-    pip uninstall -y workbench && \
-    pip cache purge
+# Copy and install dependencies (this layer only changes when deps change)
+COPY --from=deps-extractor /tmp/deps.txt /tmp/deps.txt
+RUN pip install --no-cache-dir -r /tmp/deps.txt && rm /tmp/deps.txt
 
 # Install workbench[ui] in separate layer (dependencies already satisfied)
-RUN pip install --no-cache-dir "workbench[ui]==${WORKBENCH_VERSION}"
+ARG WORKBENCH_VERSION
+RUN pip install --no-cache-dir --no-deps "workbench[ui]==${WORKBENCH_VERSION}"
 
 # Copy app code and configuration (after pip installs to avoid cache invalidation)
 COPY . /app

{workbench-0.8.228 → workbench-0.8.230}/applications/aws_dashboard/pages/models/callbacks.py

@@ -140,11 +140,14 @@ def setup_plugin_callbacks(plugins, scatter_plot_plugin: ScatterPlot, confusion_
             if isinstance(target, list):
                 target = next((t for t in target if t in inference_run), target[0])
 
+            # Check if "confidence" column exists for coloring
+            color_col = "confidence" if "confidence" in df.columns else "prediction"
+
             scatter_props = scatter_plot_plugin.update_properties(
                 df,
                 x=target,
                 y="prediction",
-                color="confidence",
+                color=color_col,
                 regression_line=True,
             )
             cm_props = [no_update] * len(confusion_matrix_plugin.properties)

{workbench-0.8.228 → workbench-0.8.230}/aws_setup/workbench_core/workbench_core/workbench_core_stack.py

@@ -180,6 +180,7 @@ class WorkbenchCoreStack(Stack):
                 "glue:GetDatabases",
                 "glue:GetTable",
                 "glue:GetTables",
+                "glue:GetPartitions",
                 "glue:SearchTables",
             ],
             resources=[f"arn:aws:glue:{self.region}:{self.account}:catalog"],

{workbench-0.8.228 → workbench-0.8.230}/aws_setup/workbench_dashboard_full/app.py

@@ -12,7 +12,7 @@ print(f"Account: {aws_account}")
 print(f"Region: {aws_region}")
 
 # When you want a different docker image change this line
-dashboard_image = "public.ecr.aws/m6i5k1r2/workbench_dashboard:v0_8_227_amd64"
+dashboard_image = "public.ecr.aws/m6i5k1r2/workbench_dashboard:v0_8_228_amd64"
 
 # Workbench Configuration
 try:

{workbench-0.8.228 → workbench-0.8.230}/aws_setup/workbench_dashboard_lite/app.py

@@ -12,7 +12,7 @@ print(f"Account: {aws_account}")
 print(f"Region: {aws_region}")
 
 # When you want a different docker image change this line
-dashboard_image = "public.ecr.aws/m6i5k1r2/workbench_dashboard:v0_8_227_amd64"
+dashboard_image = "public.ecr.aws/m6i5k1r2/workbench_dashboard:v0_8_228_amd64"
 
 # Workbench Configuration
 try:

{workbench-0.8.228 → workbench-0.8.230}/docs/admin/dashboard_docker_push.md

@@ -6,9 +6,10 @@ Notes and information on how to do the Dashboard Docker Builds and Push to AWS E
 ```
 cd applications/aws_dashboard
 vi Dockerfile
+...
 
-# Install workbench[ui] in separate layer
-RUN pip install --no-cache-dir "workbench[ui]==0.8.116"  <-- change this
+# Install workbench[ui] in separate layer (dependencies already satisfied)
+ARG WORKBENCH_VERSION=0.8.227  <-- change this
 ```
 
 ### Run the Deploy Script

{workbench-0.8.228 → workbench-0.8.230}/model_docker_images/ml_pipelines/Dockerfile

@@ -29,7 +29,7 @@ COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/pytho
 COPY --from=builder /usr/local/bin /usr/local/bin
 
 # Install workbench only
-RUN pip install --no-cache-dir workbench==0.8.223
+RUN pip install --no-cache-dir workbench==0.8.229
 
 # Add the script runner
 COPY ml_pipeline_runner.py /app/ml_pipeline_runner.py

{workbench-0.8.228 → workbench-0.8.230}/pyproject.toml

@@ -52,7 +52,7 @@ ui = [
     "dash>=3.0.0",
     "dash-bootstrap-components>=1.6.0",
     "dash-bootstrap-templates>=1.3.0",
-    "dash_ag_grid",
+    "dash_ag_grid<=32.3.2",
     "tabulate>=0.9.0",
     "matplotlib>=3.9.2",
 ]

{workbench-0.8.228 → workbench-0.8.230}/src/workbench/algorithms/dataframe/__init__.py

@@ -9,10 +9,12 @@ from .proximity import Proximity
 from .feature_space_proximity import FeatureSpaceProximity
 from .fingerprint_proximity import FingerprintProximity
 from .projection_2d import Projection2D
+from .smart_aggregator import smart_aggregator
 
 __all__ = [
     "Proximity",
     "FeatureSpaceProximity",
     "FingerprintProximity",
     "Projection2D",
+    "smart_aggregator",
 ]

workbench-0.8.230/src/workbench/algorithms/dataframe/smart_aggregator.py

@@ -0,0 +1,161 @@
+"""SmartSample: Intelligently reduce DataFrame rows by aggregating similar rows together."""
+
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import StandardScaler
+from sklearn.cluster import MiniBatchKMeans
+import logging
+
+# Set up logging
+log = logging.getLogger("workbench")
+
+
+def smart_aggregator(df: pd.DataFrame, target_rows: int = 1000, outlier_column: str = "residual") -> pd.DataFrame:
+    """
+    Reduce DataFrame rows by aggregating similar rows based on numeric column similarity.
+
+    This is a performant (2-pass) algorithm:
+    1. Pass 1: Normalize numeric columns and cluster similar rows using MiniBatchKMeans
+    2. Pass 2: Aggregate each cluster (mean for numeric, first for non-numeric)
+
+    Args:
+        df: Input DataFrame.
+        target_rows: Target number of rows in output (default: 1000).
+        outlier_column: Column where high values should resist aggregation (default: "residual").
+                       Rows with high values in this column will be kept separate while rows
+                       with low values cluster together. Set to None to disable.
+
+    Returns:
+        Reduced DataFrame with 'aggregation_count' column showing how many rows were combined.
+    """
+    if df is None or df.empty:
+        return df
+
+    n_rows = len(df)
+
+    # Preserve original column order
+    original_columns = df.columns.tolist()
+
+    # If already at or below target, just add the count column and return
+    if n_rows <= target_rows:
+        result = df.copy()
+        result["aggregation_count"] = 1
+        return result
+
+    log.info(f"smart_aggregator: Reducing {n_rows} rows to ~{target_rows} rows")
+
+    # Identify columns by type
+    df = df.copy()
+    numeric_cols = df.select_dtypes(include=["number"]).columns.tolist()
+    non_numeric_cols = [c for c in df.columns if c not in numeric_cols]
+
+    if not numeric_cols:
+        log.warning("smart_aggregator: No numeric columns for clustering, falling back to random sample")
+        result = df.sample(n=target_rows)
+        result["aggregation_count"] = 1
+        return result.reset_index(drop=True)
+
+    # Handle NaN values - fill with column median
+    df_for_clustering = df[numeric_cols].fillna(df[numeric_cols].median())
+
+    # Normalize and cluster
+    X = StandardScaler().fit_transform(df_for_clustering)
+    df["_cluster"] = MiniBatchKMeans(
+        n_clusters=min(target_rows, n_rows), random_state=42, batch_size=min(1024, n_rows), n_init=3
+    ).fit_predict(X)
+
+    # Post-process: give high-outlier rows their own unique clusters so they don't get aggregated
+    if outlier_column and outlier_column in df.columns:
+        # Top 10% of outlier values get their own clusters, capped at 200
+        n_to_isolate = min(int(n_rows * 0.1), 200)
+        threshold = df[outlier_column].nlargest(n_to_isolate).min()
+        high_outlier_mask = df[outlier_column] >= threshold
+        n_high_outliers = high_outlier_mask.sum()
+        # Assign unique cluster IDs starting after the max existing cluster
+        max_cluster = df["_cluster"].max()
+        df.loc[high_outlier_mask, "_cluster"] = range(max_cluster + 1, max_cluster + 1 + n_high_outliers)
+        log.info(f"smart_aggregator: Isolated {n_high_outliers} high-outlier rows (>= {threshold:.3f})")
+    elif outlier_column:
+        log.warning(f"smart_aggregator: outlier_column '{outlier_column}' not found in columns")
+
+    # Aggregate each cluster (mean for numeric, first for non-numeric)
+    agg_dict = {col: "mean" for col in numeric_cols} | {col: "first" for col in non_numeric_cols}
+    grouped = df.groupby("_cluster")
+    result = grouped.agg(agg_dict).reset_index(drop=True)
+    result["aggregation_count"] = grouped.size().values
+
+    # Restore original column order, with aggregation_count at the end
+    result = result[original_columns + ["aggregation_count"]]
+
+    log.info(f"smart_aggregator: Reduced to {len(result)} rows")
+    return result
+
+
+# Testing
+if __name__ == "__main__":
+    pd.set_option("display.max_columns", None)
+    pd.set_option("display.width", 1000)
+
+    # Create test data with clusters
+    np.random.seed(42)
+    n_samples = 10000
+
+    # Create 3 distinct clusters
+    cluster_1 = np.random.randn(n_samples // 3, 3) + np.array([0, 0, 0])
+    cluster_2 = np.random.randn(n_samples // 3, 3) + np.array([5, 5, 5])
+    cluster_3 = np.random.randn(n_samples // 3, 3) + np.array([10, 0, 5])
+
+    features = np.vstack([cluster_1, cluster_2, cluster_3])
+
+    # Create target and prediction columns, then compute residuals
+    target = features[:, 0] + features[:, 1] * 0.5 + np.random.randn(len(features)) * 0.1
+    prediction = target + np.random.randn(len(features)) * 0.5  # Add noise for residuals
+    residuals = np.abs(target - prediction)
+
+    data = {
+        "id": [f"id_{i}" for i in range(len(features))],
+        "A": features[:, 0],
+        "B": features[:, 1],
+        "C": features[:, 2],
+        "category": np.random.choice(["cat1", "cat2", "cat3"], len(features)),
+        "target": target,
+        "prediction": prediction,
+        "residual": residuals,
+    }
+    df = pd.DataFrame(data)
+
+    print(f"Original DataFrame: {len(df)} rows")
+    print(df.head())
+    print()
+
+    # Test smart_aggregator with residuals preservation
+    result = smart_aggregator(df, target_rows=500)
+    print(f"smart_aggregator result: {len(result)} rows")
+    print(result.head(20))
+    print()
+    print("Aggregation count stats:")
+    print(result["aggregation_count"].describe())
+    print()
+    # Show that high-residual points have lower aggregation counts
+    print("Aggregation count by residual quartile:")
+    result["residual_quartile"] = pd.qcut(result["residual"], 4, labels=["Q1 (low)", "Q2", "Q3", "Q4 (high)"])
+    print(result.groupby("residual_quartile")["aggregation_count"].mean())
+
+    # Test with real Workbench data
+    print("\n" + "=" * 80)
+    print("Testing with Workbench data...")
+    print("=" * 80)
+
+    from workbench.api import Model
+
+    model = Model("abalone-regression")
+    df = model.get_inference_predictions()
+    if df is not None:
+        print(f"\nOriginal DataFrame: {len(df)} rows")
+        print(df.head())
+
+        result = smart_aggregator(df, target_rows=500)
+        print(f"\nsmart_aggregator result: {len(result)} rows")
+        print(result.head())
+        print("\nAggregation count stats:")
+        print(result["aggregation_count"].describe())

{workbench-0.8.228 → workbench-0.8.230}/src/workbench/cached/cached_model.py

@@ -6,6 +6,7 @@ import pandas as pd
 # Workbench Imports
 from workbench.core.artifacts.model_core import ModelCore, ModelType
 from workbench.core.artifacts.cached_artifact_mixin import CachedArtifactMixin
+from workbench.algorithms.dataframe import smart_aggregator
 
 
 class CachedModel(CachedArtifactMixin, ModelCore):
@@ -85,13 +86,13 @@ class CachedModel(CachedArtifactMixin, ModelCore):
 
     @CachedArtifactMixin.cache_result
     def get_inference_predictions(
-        self, capture_name: str = "full_cross_fold", limit: int = 5000
+        self, capture_name: str = "full_cross_fold", target_rows: int = 1000
     ) -> Union[pd.DataFrame, None]:
         """Retrieve the captured prediction results for this model
 
         Args:
-            capture_name (str, optional): Specific capture_name (default: auto_inference)
-            limit (int, optional): Maximum rows to return (default: 1000)
+            capture_name (str, optional): Specific capture_name (default: full_cross_fold)
+            target_rows (int, optional): Target number of rows to return (default: 1000)
 
         Returns:
             pd.DataFrame: DataFrame of the Captured Predictions (might be None)
@@ -100,7 +101,7 @@ class CachedModel(CachedArtifactMixin, ModelCore):
         if df is None:
             return None
 
-        # Compute residual and do smart sampling based on model type
+        # Compute residual based on model type
         is_regressor = self.model_type in [ModelType.REGRESSOR, ModelType.UQ_REGRESSOR, ModelType.ENSEMBLE_REGRESSOR]
         is_classifier = self.model_type == ModelType.CLASSIFIER
 
@@ -120,21 +121,12 @@ class CachedModel(CachedArtifactMixin, ModelCore):
                     df["prediction"].map(label_to_idx).fillna(-1) - df[target].map(label_to_idx).fillna(-1)
                 )
 
-        # Smart sampling: half high-residual rows, half random from the rest
-        if "residual" in df.columns and len(df) > limit:
-            half_limit = limit // 2
-            self.log.warning(
-                f"{self.name}:{capture_name} Sampling {limit} rows (top {half_limit} residuals + {half_limit} random)"
+        # Use smart_aggregator to aggregate similar rows if we have too many
+        if len(df) > target_rows:
+            self.log.info(
+                f"{self.name}:{capture_name} Using smart_aggregator to reduce {len(df)} rows to ~{target_rows}"
             )
-            top_residuals = df.nlargest(half_limit, "residual")
-            remaining = df.drop(top_residuals.index)
-            random_sample = remaining.sample(min(half_limit, len(remaining)))
-            return pd.concat([top_residuals, random_sample]).reset_index(drop=True)
-
-        # Fallback: just limit rows if no residual computed
-        if len(df) > limit:
-            self.log.warning(f"{self.name}:{capture_name} Sampling to {limit} rows")
-            return df.sample(limit)
+            df = smart_aggregator(df, target_rows=target_rows)
 
         return df
 

{workbench-0.8.228/src/workbench/model_scripts/pytorch_model → workbench-0.8.230/src/workbench/model_script_utils}/model_script_utils.py

@@ -249,6 +249,36 @@ def output_fn(output_df: pd.DataFrame, accept_type: str) -> tuple[str, str]:
         raise RuntimeError(f"{accept_type} accept type is not supported by this script.")
 
 
+def cap_std_outliers(std_array: np.ndarray) -> np.ndarray:
+    """Cap extreme outliers in prediction_std using IQR method.
+
+    Uses the standard IQR fence (Q3 + 1.5*IQR) to cap extreme values.
+    This prevents unreasonably large std values while preserving the
+    relative ordering and keeping meaningful high-uncertainty signals.
+
+    Args:
+        std_array: Array of standard deviations (n_samples,) or (n_samples, n_targets)
+
+    Returns:
+        Array with outliers capped at the upper fence
+    """
+    if std_array.ndim == 1:
+        std_array = std_array.reshape(-1, 1)
+        squeeze = True
+    else:
+        squeeze = False
+
+    capped = std_array.copy()
+    for col in range(capped.shape[1]):
+        col_data = capped[:, col]
+        q1, q3 = np.percentile(col_data, [25, 75])
+        iqr = q3 - q1
+        upper_bound = q3 + 1.5 * iqr
+        capped[:, col] = np.minimum(col_data, upper_bound)
+
+    return capped.squeeze() if squeeze else capped
+
+
 def compute_regression_metrics(y_true: np.ndarray, y_pred: np.ndarray) -> dict[str, float]:
     """Compute standard regression metrics.
 

{workbench-0.8.228 → workbench-0.8.230}/src/workbench/model_scripts/chemprop/chemprop.template

@@ -20,6 +20,7 @@ import torch
 from chemprop import data, models
 
 from model_script_utils import (
+    cap_std_outliers,
     expand_proba_column,
     input_fn,
     output_fn,
@@ -245,6 +246,7 @@ def predict_fn(df: pd.DataFrame, model_dict: dict) -> pd.DataFrame:
     preds_std = np.std(np.stack(all_preds), axis=0)
     if preds.ndim == 1:
         preds, preds_std = preds.reshape(-1, 1), preds_std.reshape(-1, 1)
+    preds_std = cap_std_outliers(preds_std)
 
     print(f"Inference complete: {preds.shape[0]} predictions")
 
@@ -701,6 +703,7 @@ if __name__ == "__main__":
             preds_std = np.std(np.stack(all_ens_preds), axis=0)
             if preds_std.ndim == 1:
                 preds_std = preds_std.reshape(-1, 1)
+            preds_std = cap_std_outliers(preds_std)
 
         print("\n--- Per-target metrics ---")
         for t_idx, t_name in enumerate(target_columns):

{workbench-0.8.228 → workbench-0.8.230}/src/workbench/model_scripts/chemprop/generated_model_script.py

@@ -57,11 +57,11 @@ DEFAULT_HYPERPARAMETERS = {
 
 # Template parameters (filled in by Workbench)
 TEMPLATE_PARAMS = {
-    "model_type": "classifier",
-    "targets": ['class'],
+    "model_type": "uq_regressor",
+    "targets": ['udm_asy_res_value'],
     "feature_list": ['smiles'],
     "id_column": "udm_mol_bat_id",
-    "model_metrics_s3_path": "s3://ideaya-sageworks-bucket/models/logd-value-class-chemprop-1-dt/training",
+    "model_metrics_s3_path": "s3://ideaya-sageworks-bucket/models/logd-value-reg-chemprop-1-dt/training",
     "hyperparameters": {},
 }
 

{workbench-0.8.228/src/workbench/model_script_utils → workbench-0.8.230/src/workbench/model_scripts/chemprop}/model_script_utils.py

@@ -249,6 +249,36 @@ def output_fn(output_df: pd.DataFrame, accept_type: str) -> tuple[str, str]:
         raise RuntimeError(f"{accept_type} accept type is not supported by this script.")
 
 
+def cap_std_outliers(std_array: np.ndarray) -> np.ndarray:
+    """Cap extreme outliers in prediction_std using IQR method.
+
+    Uses the standard IQR fence (Q3 + 1.5*IQR) to cap extreme values.
+    This prevents unreasonably large std values while preserving the
+    relative ordering and keeping meaningful high-uncertainty signals.
+
+    Args:
+        std_array: Array of standard deviations (n_samples,) or (n_samples, n_targets)
+
+    Returns:
+        Array with outliers capped at the upper fence
+    """
+    if std_array.ndim == 1:
+        std_array = std_array.reshape(-1, 1)
+        squeeze = True
+    else:
+        squeeze = False
+
+    capped = std_array.copy()
+    for col in range(capped.shape[1]):
+        col_data = capped[:, col]
+        q1, q3 = np.percentile(col_data, [25, 75])
+        iqr = q3 - q1
+        upper_bound = q3 + 1.5 * iqr
+        capped[:, col] = np.minimum(col_data, upper_bound)
+
+    return capped.squeeze() if squeeze else capped
+
+
 def compute_regression_metrics(y_true: np.ndarray, y_pred: np.ndarray) -> dict[str, float]:
     """Compute standard regression metrics.
 

{workbench-0.8.228 → workbench-0.8.230}/src/workbench/model_scripts/pytorch_model/generated_model_script.py

@@ -64,7 +64,7 @@ TEMPLATE_PARAMS = {
     "features": ['chi2v', 'fr_sulfone', 'chi1v', 'bcut2d_logplow', 'fr_piperzine', 'kappa3', 'smr_vsa1', 'slogp_vsa5', 'fr_ketone_topliss', 'fr_sulfonamd', 'fr_imine', 'fr_benzene', 'fr_ester', 'chi2n', 'labuteasa', 'peoe_vsa2', 'smr_vsa6', 'bcut2d_chglo', 'fr_sh', 'peoe_vsa1', 'fr_allylic_oxid', 'chi4n', 'fr_ar_oh', 'fr_nh0', 'fr_term_acetylene', 'slogp_vsa7', 'slogp_vsa4', 'estate_vsa1', 'vsa_estate4', 'numbridgeheadatoms', 'numheterocycles', 'fr_ketone', 'fr_morpholine', 'fr_guanido', 'estate_vsa2', 'numheteroatoms', 'fr_nitro_arom_nonortho', 'fr_piperdine', 'nocount', 'numspiroatoms', 'fr_aniline', 'fr_thiophene', 'slogp_vsa10', 'fr_amide', 'slogp_vsa2', 'fr_epoxide', 'vsa_estate7', 'fr_ar_coo', 'fr_imidazole', 'fr_nitrile', 'fr_oxazole', 'numsaturatedrings', 'fr_pyridine', 'fr_hoccn', 'fr_ndealkylation1', 'numaliphaticheterocycles', 'fr_phenol', 'maxpartialcharge', 'vsa_estate5', 'peoe_vsa13', 'minpartialcharge', 'qed', 'fr_al_oh', 'slogp_vsa11', 'chi0n', 'fr_bicyclic', 'peoe_vsa12', 'fpdensitymorgan1', 'fr_oxime', 'molwt', 'fr_dihydropyridine', 'smr_vsa5', 'peoe_vsa5', 'fr_nitro', 'hallkieralpha', 'heavyatommolwt', 'fr_alkyl_halide', 'peoe_vsa8', 'fr_nhpyrrole', 'fr_isocyan', 'bcut2d_chghi', 'fr_lactam', 'peoe_vsa11', 'smr_vsa9', 'tpsa', 'chi4v', 'slogp_vsa1', 'phi', 'bcut2d_logphi', 'avgipc', 'estate_vsa11', 'fr_coo', 'bcut2d_mwhi', 'numunspecifiedatomstereocenters', 'vsa_estate10', 'estate_vsa8', 'numvalenceelectrons', 'fr_nh2', 'fr_lactone', 'vsa_estate1', 'estate_vsa4', 'numatomstereocenters', 'vsa_estate8', 'fr_para_hydroxylation', 'peoe_vsa3', 'fr_thiazole', 'peoe_vsa10', 'fr_ndealkylation2', 'slogp_vsa12', 'peoe_vsa9', 'maxestateindex', 'fr_quatn', 'smr_vsa7', 'minestateindex', 'numaromaticheterocycles', 'numrotatablebonds', 'fr_ar_nh', 'fr_ether', 'exactmolwt', 'fr_phenol_noorthohbond', 'slogp_vsa3', 'fr_ar_n', 'sps', 'fr_c_o_nocoo', 'bertzct', 'peoe_vsa7', 'slogp_vsa8', 'numradicalelectrons', 'molmr', 'fr_tetrazole', 'numsaturatedcarbocycles', 'bcut2d_mrhi', 'kappa1', 'numamidebonds', 'fpdensitymorgan2', 'smr_vsa8', 'chi1n', 'estate_vsa6', 'fr_barbitur', 'fr_diazo', 'kappa2', 'chi0', 'bcut2d_mrlow', 'balabanj', 'peoe_vsa4', 'numhacceptors', 'fr_sulfide', 'chi3n', 'smr_vsa2', 'fr_al_oh_notert', 'fr_benzodiazepine', 'fr_phos_ester', 'fr_aldehyde', 'fr_coo2', 'estate_vsa5', 'fr_prisulfonamd', 'numaromaticcarbocycles', 'fr_unbrch_alkane', 'fr_urea', 'fr_nitroso', 'smr_vsa10', 'fr_c_s', 'smr_vsa3', 'fr_methoxy', 'maxabspartialcharge', 'slogp_vsa9', 'heavyatomcount', 'fr_azide', 'chi3v', 'smr_vsa4', 'mollogp', 'chi0v', 'fr_aryl_methyl', 'fr_nh1', 'fpdensitymorgan3', 'fr_furan', 'fr_hdrzine', 'fr_arn', 'numaromaticrings', 'vsa_estate3', 'fr_azo', 'fr_halogen', 'estate_vsa9', 'fr_hdrzone', 'numhdonors', 'fr_alkyl_carbamate', 'fr_isothiocyan', 'minabspartialcharge', 'fr_al_coo', 'ringcount', 'chi1', 'estate_vsa7', 'fr_nitro_arom', 'vsa_estate9', 'minabsestateindex', 'maxabsestateindex', 'vsa_estate6', 'estate_vsa10', 'estate_vsa3', 'fr_n_o', 'fr_amidine', 'fr_thiocyan', 'fr_phos_acid', 'fr_c_o', 'fr_imide', 'numaliphaticrings', 'peoe_vsa6', 'vsa_estate2', 'nhohcount', 'numsaturatedheterocycles', 'slogp_vsa6', 'peoe_vsa14', 'fractioncsp3', 'bcut2d_mwlow', 'numaliphaticcarbocycles', 'fr_priamide', 'nacid', 'nbase', 'naromatom', 'narombond', 'sz', 'sm', 'sv', 'sse', 'spe', 'sare', 'sp', 'si', 'mz', 'mm', 'mv', 'mse', 'mpe', 'mare', 'mp', 'mi', 'xch_3d', 'xch_4d', 'xch_5d', 'xch_6d', 'xch_7d', 'xch_3dv', 'xch_4dv', 'xch_5dv', 'xch_6dv', 'xch_7dv', 'xc_3d', 'xc_4d', 'xc_5d', 'xc_6d', 'xc_3dv', 'xc_4dv', 'xc_5dv', 'xc_6dv', 'xpc_4d', 'xpc_5d', 'xpc_6d', 'xpc_4dv', 'xpc_5dv', 'xpc_6dv', 'xp_0d', 'xp_1d', 'xp_2d', 'xp_3d', 'xp_4d', 'xp_5d', 'xp_6d', 'xp_7d', 'axp_0d', 'axp_1d', 'axp_2d', 'axp_3d', 'axp_4d', 'axp_5d', 'axp_6d', 'axp_7d', 'xp_0dv', 'xp_1dv', 'xp_2dv', 'xp_3dv', 'xp_4dv', 'xp_5dv', 'xp_6dv', 'xp_7dv', 'axp_0dv', 'axp_1dv', 'axp_2dv', 'axp_3dv', 'axp_4dv', 'axp_5dv', 'axp_6dv', 'axp_7dv', 'c1sp1', 'c2sp1', 'c1sp2', 'c2sp2', 'c3sp2', 'c1sp3', 'c2sp3', 'c3sp3', 'c4sp3', 'hybratio', 'fcsp3', 'num_stereocenters', 'num_unspecified_stereocenters', 'num_defined_stereocenters', 'num_r_centers', 'num_s_centers', 'num_stereobonds', 'num_e_bonds', 'num_z_bonds', 'stereo_complexity', 'frac_defined_stereo'],
     "id_column": "udm_mol_bat_id",
     "compressed_features": [],
-    "model_metrics_s3_path": "s3://ideaya-sageworks-bucket/models/logd-value-class-pytorch-1-dt/training",
+    "model_metrics_s3_path": "s3://ideaya-sageworks-bucket/models/caco2-er-class-pytorch-1-fr/training",
     "hyperparameters": {},
 }
 
@@ -152,24 +152,30 @@ def predict_fn(df: pd.DataFrame, model_dict: dict) -> pd.DataFrame:
         print("Decompressing features for prediction...")
         matched_df, features = decompress_features(matched_df, features, compressed_features)
 
-    # Track missing features
-    missing_mask = matched_df[features].isna().any(axis=1)
-    if missing_mask.any():
-        print(f"Warning: {missing_mask.sum()} rows have missing features")
+    # Impute missing values (categorical with mode, continuous handled by scaler)
+    missing_counts = matched_df[features].isna().sum()
+    if missing_counts.any():
+        missing_features = missing_counts[missing_counts > 0]
+        print(f"Imputing missing values: {missing_features.to_dict()}")
+
+        # Load categorical imputation values if available
+        impute_path = os.path.join(model_dir, "categorical_impute.json")
+        if os.path.exists(impute_path):
+            with open(impute_path) as f:
+                cat_impute_values = json.load(f)
+            for col in categorical_cols:
+                if col in cat_impute_values and matched_df[col].isna().any():
+                    matched_df[col] = matched_df[col].fillna(cat_impute_values[col])
+        # Continuous features are imputed by FeatureScaler.transform() using column means
 
     # Initialize output columns
     df["prediction"] = np.nan
     if model_type in ["regressor", "uq_regressor"]:
         df["prediction_std"] = np.nan
 
-    complete_df = matched_df[~missing_mask].copy()
-    if len(complete_df) == 0:
-        print("Warning: No complete rows to predict on")
-        return df
-
-    # Prepare data for inference (with standardization)
+    # Prepare data for inference (with standardization and continuous imputation)
     x_cont, x_cat, _, _, _ = prepare_data(
-        complete_df, continuous_cols, categorical_cols, category_mappings=category_mappings, scaler=scaler
+        matched_df, continuous_cols, categorical_cols, category_mappings=category_mappings, scaler=scaler
     )
 
     # Collect ensemble predictions
@@ -191,28 +197,20 @@ def predict_fn(df: pd.DataFrame, model_dict: dict) -> pd.DataFrame:
         class_preds = np.argmax(avg_probs, axis=1)
         predictions = label_encoder.inverse_transform(class_preds)
 
-        all_proba = pd.Series([None] * len(df), index=df.index, dtype=object)
-        all_proba.loc[~missing_mask] = [p.tolist() for p in avg_probs]
-        df["pred_proba"] = all_proba
+        df["pred_proba"] = [p.tolist() for p in avg_probs]
         df = expand_proba_column(df, label_encoder.classes_)
     else:
         # Regression
         predictions = preds.flatten()
-        df.loc[~missing_mask, "prediction_std"] = preds_std.flatten()
+        df["prediction_std"] = preds_std.flatten()
 
         # Add UQ intervals if available
         if uq_models and uq_metadata:
-            X_complete = complete_df[features]
-            df_complete = df.loc[~missing_mask].copy()
-            df_complete["prediction"] = predictions  # Set prediction before compute_confidence
-            df_complete = predict_intervals(df_complete, X_complete, uq_models, uq_metadata)
-            df_complete = compute_confidence(df_complete, uq_metadata["median_interval_width"], "q_10", "q_90")
-            # Copy UQ columns back to main dataframe
-            for col in df_complete.columns:
-                if col.startswith("q_") or col == "confidence":
-                    df.loc[~missing_mask, col] = df_complete[col].values
-
-    df.loc[~missing_mask, "prediction"] = predictions
+            df["prediction"] = predictions  # Set prediction before compute_confidence
+            df = predict_intervals(df, matched_df[features], uq_models, uq_metadata)
+            df = compute_confidence(df, uq_metadata["median_interval_width"], "q_10", "q_90")
+
+    df["prediction"] = predictions
     return df
 
 
@@ -275,11 +273,11 @@ if __name__ == "__main__":
     all_df = pd.concat([pd.read_csv(f, engine="python") for f in training_files])
     check_dataframe(all_df, "training_df")
 
-    # Drop rows with missing features
+    # Drop rows with missing target (required for training)
     initial_count = len(all_df)
-    all_df = all_df.dropna(subset=features)
+    all_df = all_df.dropna(subset=[target])
     if len(all_df) < initial_count:
-        print(f"Dropped {initial_count - len(all_df)} rows with missing features")
+        print(f"Dropped {initial_count - len(all_df)} rows with missing target")
 
     print(f"Target: {target}")
     print(f"Features: {features}")
@@ -301,6 +299,23 @@ if __name__ == "__main__":
     print(f"Categorical: {categorical_cols}")
     print(f"Continuous: {len(continuous_cols)} columns")
 
+    # Report and handle missing values in features
+    # Compute categorical imputation values (mode) for use at inference time
+    cat_impute_values = {}
+    for col in categorical_cols:
+        mode_val = all_df[col].mode().iloc[0] if not all_df[col].mode().empty else all_df[col].cat.categories[0]
+        cat_impute_values[col] = str(mode_val)  # Convert to string for JSON serialization
+
+    missing_counts = all_df[features].isna().sum()
+    if missing_counts.any():
+        missing_features = missing_counts[missing_counts > 0]
+        print(f"Missing values in features (will be imputed): {missing_features.to_dict()}")
+        # Impute categorical features with mode (most frequent value)
+        for col in categorical_cols:
+            if all_df[col].isna().any():
+                all_df[col] = all_df[col].fillna(cat_impute_values[col])
+        # Continuous features are imputed by FeatureScaler.transform() using column means
+
     # -------------------------------------------------------------------------
     # Classification setup
     # -------------------------------------------------------------------------
@@ -506,6 +521,9 @@ if __name__ == "__main__":
     with open(os.path.join(args.model_dir, "feature_metadata.json"), "w") as f:
         json.dump({"continuous_cols": continuous_cols, "categorical_cols": categorical_cols}, f)
 
+    with open(os.path.join(args.model_dir, "categorical_impute.json"), "w") as f:
+        json.dump(cat_impute_values, f)
+
     with open(os.path.join(args.model_dir, "hyperparameters.json"), "w") as f:
         json.dump(hyperparameters, f, indent=2)