workbench 0.8.233__tar.gz → 0.8.241__tar.gz

{workbench-0.8.233 → workbench-0.8.241}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: workbench
-Version: 0.8.233
+Version: 0.8.241
 Summary: Workbench: A Dashboard and Python API for creating and deploying AWS SageMaker Model Pipelines
 Author-email: SuperCowPowers LLC <support@supercowpowers.com>
 License: MIT License
@@ -40,7 +40,7 @@ Requires-Dist: boto3>=1.31.76
 Requires-Dist: botocore>=1.31.76
 Requires-Dist: redis>=5.0.1
 Requires-Dist: numpy>=1.26.4
-Requires-Dist: pandas>=2.2.1
+Requires-Dist: pandas<3.0,>=2.2.1
 Requires-Dist: awswrangler>=3.4.0
 Requires-Dist: sagemaker<3.0,>=2.143
 Requires-Dist: cryptography>=44.0.2

{workbench-0.8.233 → workbench-0.8.241}/applications/aws_dashboard/Dockerfile

@@ -1,5 +1,5 @@
 # Define workbench version once at the top
-ARG WORKBENCH_VERSION=0.8.232
+ARG WORKBENCH_VERSION=0.8.237
 
 # Stage 1: Extract dependencies from workbench[ui]
 FROM python:3.12.5-bookworm AS deps-extractor

{workbench-0.8.233 → workbench-0.8.241}/applications/aws_dashboard/app.py

@@ -33,6 +33,7 @@ app = Dash(
     title=app_title,
     use_pages=True,
     external_stylesheets=css_files,
+    assets_folder="assets",
 )
 
 # Register the CSS route in the ThemeManager
@@ -90,6 +91,8 @@ app.layout = html.Div(
         # URL for subpage navigation (jumping to feature_sets, models, etc.)
         dcc.Location(id="url", refresh="callback-nav"),
         dcc.Store(id=plugin_info_id, data={}),
+        # Theme store for dynamic theme switching (plugins can use this as Input to re-render figures)
+        dcc.Store(id="workbench-theme-store", data=tm.current_theme()),
         # Settings menu in top-right corner
         html.Div(
             settings_menu.create_component(settings_menu_id),
@@ -100,10 +103,10 @@ app.layout = html.Div(
     **{"data-bs-theme": tm.data_bs_theme()},
 )
 
-# Clientside callback for theme switching (stores in localStorage, sets cookie, reloads page)
+# Clientside callback for theme switching (stores in localStorage, sets cookie, triggers checkmark update)
 app.clientside_callback(
-    settings_menu.get_clientside_callback_code(settings_menu_id),
-    Output(f"{settings_menu_id}-dummy", "data"),
+    settings_menu.get_clientside_callback_code(),
+    Output(f"{settings_menu_id}-init", "data"),
     Input({"type": f"{settings_menu_id}-theme-item", "theme": ALL}, "n_clicks"),
     State({"type": f"{settings_menu_id}-theme-item", "theme": ALL}, "id"),
 )
@@ -116,6 +119,13 @@ app.clientside_callback(
     State({"type": f"{settings_menu_id}-checkmark", "theme": ALL}, "id"),
 )
 
+# Clientside callback to update the workbench-theme-store (plugins listen to this for figure re-renders)
+app.clientside_callback(
+    "function(theme) { return theme; }",
+    Output("workbench-theme-store", "data"),
+    Input(f"{settings_menu_id}-init", "data"),
+)
+
 # Spin up the Plugin Manager
 pm = PluginManager()
 

{workbench-0.8.233 → workbench-0.8.241}/applications/aws_dashboard/pages/endpoints/callbacks.py

@@ -14,6 +14,9 @@ from workbench.cached.cached_endpoint import CachedEndpoint
 # Get the Workbench logger
 log = logging.getLogger("workbench")
 
+# Theme store ID (defined in app.py)
+THEME_STORE_ID = "workbench-theme-store"
+
 
 def on_page_load():
     @callback(
@@ -119,3 +122,23 @@ def setup_plugin_callbacks(plugins):
 
         # Return all the updated properties
         return all_props
+
+
+def setup_theme_callback(plugins):
+    """Set up a callback to update all plugins when the theme changes.
+
+    Args:
+        plugins: List of all plugin instances on this page.
+    """
+
+    @callback(
+        [Output(cid, prop, allow_duplicate=True) for p in plugins for cid, prop in p.properties],
+        Input(THEME_STORE_ID, "data"),
+        prevent_initial_call=True,
+    )
+    def on_theme_change(theme):
+        """Update all plugins when the theme changes."""
+        all_props = []
+        for plugin in plugins:
+            all_props.extend(plugin.set_theme(theme))
+        return all_props

{workbench-0.8.233 → workbench-0.8.241}/applications/aws_dashboard/pages/endpoints/page.py

@@ -69,3 +69,4 @@ callbacks.update_endpoint_metrics(endpoints_view)
 # For all the plugins we have we'll call their update_properties method
 if plugins:
     callbacks.setup_plugin_callbacks(plugins)
+    callbacks.setup_theme_callback(plugins)

workbench-0.8.241/applications/aws_dashboard/pages/models/callbacks.py

@@ -0,0 +1,137 @@
+"""Callbacks for the Model Subpage Web User Interface"""
+
+import logging
+from dash import callback, Input, Output, State
+from dash.exceptions import PreventUpdate
+from urllib.parse import urlparse, parse_qs
+
+# Workbench Imports
+from workbench.web_interface.page_views.models_page_view import ModelsPageView
+from workbench.web_interface.components.plugins.ag_table import AGTable
+from workbench.cached.cached_model import CachedModel
+
+# Get the Workbench logger
+log = logging.getLogger("workbench")
+
+# Theme store ID (defined in app.py)
+THEME_STORE_ID = "workbench-theme-store"
+
+
+def on_page_load():
+    @callback(
+        Output("models_table", "selectedRows"),
+        Output("models_page_loaded", "data"),
+        Input("url", "href"),
+        Input("models_table", "rowData"),
+        State("models_page_loaded", "data"),
+        prevent_initial_call=True,
+    )
+    def _on_page_load(href, row_data, page_already_loaded):
+        if page_already_loaded:
+            raise PreventUpdate
+
+        if not href or not row_data:
+            raise PreventUpdate
+
+        parsed = urlparse(href)
+        if parsed.path != "/models":
+            raise PreventUpdate
+
+        selected_name = parse_qs(parsed.query).get("name", [None])[0]
+        if not selected_name:
+            return [row_data[0]], True
+
+        for row in row_data:
+            if row.get("name") == selected_name:
+                return [row], True
+
+        raise PreventUpdate
+
+
+def model_table_refresh(page_view: ModelsPageView, table: AGTable):
+    @callback(
+        [Output(component_id, prop) for component_id, prop in table.properties],
+        Input("models_refresh", "n_intervals"),
+    )
+    def _model_table_refresh(_n):
+        """Return the table data for the Models Table"""
+        page_view.refresh()
+        models = page_view.models()
+        models["name"] = models["Model Group"]
+        models["id"] = range(len(models))
+        return table.update_properties(models)
+
+
+def setup_plugin_callbacks(plugins, model_details_plugin):
+    """Set up callbacks for all plugins.
+
+    Args:
+        plugins: List of all plugins (model_details, shap_plot, model_plot)
+        model_details_plugin: The model_details plugin (handles the inference dropdown)
+    """
+
+    # Register internal callbacks for all plugins
+    for plugin in plugins:
+        plugin.register_internal_callbacks()
+
+    # Separate model_details from other plugins - it sets the dropdown that others depend on
+    other_plugins = [p for p in plugins if p is not model_details_plugin]
+
+    # Callback to update model_details when a model is selected (this sets the dropdown)
+    @callback(
+        [Output(cid, prop) for cid, prop in model_details_plugin.properties],
+        Input("models_table", "selectedRows"),
+        State("model_details-dropdown", "value"),
+        State("model_details-header", "children"),
+    )
+    def update_model_details(selected_rows, inference_run, previous_model_name):
+        """Update model_details when a model is selected."""
+        if not selected_rows or selected_rows[0] is None:
+            raise PreventUpdate
+
+        model_name = selected_rows[0]["name"]
+        model = CachedModel(model_name)
+
+        return model_details_plugin.update_properties(
+            model, inference_run=inference_run, previous_model_name=previous_model_name
+        )
+
+    # Callback to update other plugins when the dropdown changes (triggered by model_details)
+    @callback(
+        [Output(cid, prop) for p in other_plugins for cid, prop in p.properties],
+        Input("model_details-dropdown", "value"),
+        State("models_table", "selectedRows"),
+    )
+    def update_other_plugins(inference_run, selected_rows):
+        """Update other plugins when inference run changes."""
+        if not selected_rows or selected_rows[0] is None or inference_run is None:
+            raise PreventUpdate
+
+        model_name = selected_rows[0]["name"]
+        model = CachedModel(model_name)
+
+        all_props = []
+        for plugin in other_plugins:
+            all_props.extend(plugin.update_properties(model, inference_run=inference_run))
+
+        return all_props
+
+
+def setup_theme_callback(plugins):
+    """Set up a callback to update all plugins when the theme changes.
+
+    Args:
+        plugins: List of all plugin instances on this page.
+    """
+
+    @callback(
+        [Output(cid, prop, allow_duplicate=True) for p in plugins for cid, prop in p.properties],
+        Input(THEME_STORE_ID, "data"),
+        prevent_initial_call=True,
+    )
+    def on_theme_change(theme):
+        """Update all plugins when the theme changes."""
+        all_props = []
+        for plugin in plugins:
+            all_props.extend(plugin.set_theme(theme))
+        return all_props

{workbench-0.8.233 → workbench-0.8.241}/applications/aws_dashboard/pages/models/page.py

@@ -1,19 +1,13 @@
 """Models:  A Workbench Web Interface to view, and interact with Models"""
 
-from dash import register_page, html
+from dash import register_page
 
 # Local Imports
 from .layout import models_layout
 from . import callbacks
 
 # Workbench Imports
-from workbench.web_interface.components.plugins import (
-    model_details,
-    ag_table,
-    shap_summary_plot,
-    scatter_plot,
-    confusion_matrix,
-)
+from workbench.web_interface.components.plugins import model_details, ag_table, shap_summary_plot, model_plot
 from workbench.web_interface.page_views.models_page_view import ModelsPageView
 
 # Register this page with Dash
@@ -31,35 +25,19 @@ models_table_component = models_table.create_component("models_table", max_heigh
 my_model_details = model_details.ModelDetails()
 model_details_component = my_model_details.create_component("model_details")
 
-# Create ScatterPlot plugin for regression models
-my_scatter_plot = scatter_plot.ScatterPlot()
-scatter_plot_component = my_scatter_plot.create_component("model_scatter_plot")
-
-# Create ConfusionMatrix plugin for classification models
-my_confusion_matrix = confusion_matrix.ConfusionMatrix()
-confusion_matrix_component = my_confusion_matrix.create_component("model_confusion_matrix")
-
-# Wrap both in visibility-controlled containers
-# Show scatter plot by default (with "Waiting for Data..." placeholder)
-# The callback will toggle visibility based on model type once data loads
-model_plot_container = html.Div(
-    [
-        html.Div(scatter_plot_component, id="scatter-plot-container", style={"display": "block"}),
-        html.Div(confusion_matrix_component, id="confusion-matrix-container", style={"display": "none"}),
-    ],
-    id="model-plot-container",
-)
+# Create ModelPlot plugin (handles both regression scatter plot and classification confusion matrix)
+my_model_plot = model_plot.ModelPlot()
+model_plot_component = my_model_plot.create_component("model_plot")
 
 # Shap summary plot component
 my_shap_plot = shap_summary_plot.ShapSummaryPlot()
 shap_plot_component = my_shap_plot.create_component("shap_plot")
 
-
 # Capture our components in a dictionary to send off to the layout
 components = {
     "models_table": models_table_component,
     "model_details": model_details_component,
-    "model_plot": model_plot_container,
+    "model_plot": model_plot_component,
     "shap_plot": shap_plot_component,
 }
 
@@ -76,7 +54,10 @@ callbacks.on_page_load()
 callbacks.model_table_refresh(model_view, models_table)
 
 # All model visualization plugins
-plugins = [my_model_details, my_shap_plot, my_scatter_plot, my_confusion_matrix]
+plugins = [my_model_details, my_shap_plot, my_model_plot]
+
+# Set up callbacks for all the plugins
+callbacks.setup_plugin_callbacks(plugins, my_model_details)
 
-# Set up callbacks for all the plugins (includes scatter plot and confusion matrix)
-callbacks.setup_plugin_callbacks(plugins, my_scatter_plot, my_confusion_matrix)
+# Set up theme change callback for all plugins
+callbacks.setup_theme_callback(plugins)

{workbench-0.8.233 → workbench-0.8.241}/aws_setup/workbench_compute/workbench_compute/lambdas/batch_failure/handler.py

@@ -42,7 +42,7 @@ def get_log_events(log_stream_name: str, limit: int = 50) -> list[str]:
         return [f"(Error fetching logs: {e})"]
 
 
-def format_log_messages(messages: list[str], max_chars: int = 3000) -> str:
+def format_log_messages(messages: list[str], max_chars: int = 10000) -> str:
     """
     Format log messages for SNS, truncating if necessary.
 
@@ -97,7 +97,7 @@ def lambda_handler(event, context):
     # Fetch CloudWatch logs
     log_messages = []
     if log_stream_name:
-        log_messages = get_log_events(log_stream_name)
+        log_messages = get_log_events(log_stream_name, limit=200)
 
     # Build the console URL
     console_url = f"https://{region}.console.aws.amazon.com/batch/home?region={region}" f"#jobs/fargate/detail/{job_id}"

{workbench-0.8.233 → workbench-0.8.241}/aws_setup/workbench_compute/workbench_compute/workbench_compute_stack.py

@@ -13,6 +13,8 @@ from aws_cdk import (
     aws_events as events,
     aws_events_targets as targets,
     aws_logs as logs,
+    aws_cloudwatch as cloudwatch,
+    aws_cloudwatch_actions as cw_actions,
     Duration,
     Size,
 )
@@ -67,7 +69,7 @@ class WorkbenchComputeStack(Stack):
         self.batch_job_definitions = self.create_batch_job_definitions()
 
         # Create ML Pipeline SQS Queue and Lambda (after Batch resources)
-        self.ml_pipeline_queue = self.create_ml_pipeline_queue()
+        self.ml_pipeline_dlq, self.ml_pipeline_queue = self.create_ml_pipeline_queue()
         self.batch_trigger_lambda = self.create_batch_trigger_lambda()
 
         # Create Batch failure notification resources
@@ -75,6 +77,9 @@ class WorkbenchComputeStack(Stack):
         self.batch_failure_lambda = self.create_batch_failure_lambda()
         self.create_batch_failure_rule()
 
+        # Create DLQ alarm (uses the batch failure topic for notifications)
+        self.create_dlq_alarm()
+
     #####################
     #   Batch Compute   #
     #####################
@@ -166,8 +171,12 @@ class WorkbenchComputeStack(Stack):
     ###########################
     #  ML Pipeline SQS Queue  #
     ###########################
-    def create_ml_pipeline_queue(self) -> sqs.Queue:
-        """Create SQS FIFO queue for ML pipeline orchestration with deduplication."""
+    def create_ml_pipeline_queue(self) -> tuple[sqs.Queue, sqs.Queue]:
+        """Create SQS FIFO queue for ML pipeline orchestration with deduplication.
+
+        Returns:
+            Tuple of (dlq, main_queue)
+        """
 
         # Dead letter queue for failed messages (also FIFO)
         dlq = sqs.Queue(
@@ -179,7 +188,7 @@ class WorkbenchComputeStack(Stack):
         )
 
         # Main FIFO queue with deduplication
-        return sqs.Queue(
+        main_queue = sqs.Queue(
             self,
             "MLPipelineQueue",
             queue_name="workbench-ml-pipeline-queue.fifo",  # Must end with .fifo
@@ -190,6 +199,8 @@ class WorkbenchComputeStack(Stack):
             dead_letter_queue=sqs.DeadLetterQueue(max_receive_count=1, queue=dlq),
         )
 
+        return dlq, main_queue
+
     def create_batch_trigger_lambda(self) -> lambda_.Function:
         """Create Lambda function to process SQS messages and trigger Batch jobs.
 
@@ -237,13 +248,27 @@ class WorkbenchComputeStack(Stack):
     ################################
     def create_batch_failure_topic(self) -> sns.Topic:
         """Create SNS topic for batch job failure notifications."""
-        return sns.Topic(
+        topic = sns.Topic(
             self,
             "BatchJobFailureTopic",
             topic_name="workbench-batch-job-failure",
             display_name="Workbench Batch Job Failure",
         )
 
+        # Allow CloudWatch Alarms to publish to this topic
+        topic.add_to_resource_policy(
+            iam.PolicyStatement(
+                sid="AllowCloudWatchAlarms",
+                effect=iam.Effect.ALLOW,
+                principals=[iam.ServicePrincipal("cloudwatch.amazonaws.com")],
+                actions=["sns:Publish"],
+                resources=[topic.topic_arn],
+                conditions={"ArnLike": {"aws:SourceArn": f"arn:aws:cloudwatch:{self.region}:{self.account}:alarm:*"}},
+            )
+        )
+
+        return topic
+
     def create_batch_failure_lambda(self) -> lambda_.Function:
         """Create Lambda function to handle batch job failures and send notifications."""
 
@@ -300,3 +325,26 @@ class WorkbenchComputeStack(Stack):
         batch_failure_rule.add_target(targets.LambdaFunction(self.batch_failure_lambda))
 
         return batch_failure_rule
+
+    def create_dlq_alarm(self) -> cloudwatch.Alarm:
+        """Create CloudWatch alarm for DLQ messages to alert on pipeline failures."""
+
+        alarm = cloudwatch.Alarm(
+            self,
+            "MLPipelineDLQAlarm",
+            alarm_name="workbench-ml-pipeline-dlq-alarm",
+            alarm_description="Alert when messages appear in the ML Pipeline dead letter queue",
+            metric=self.ml_pipeline_dlq.metric_approximate_number_of_messages_visible(
+                period=Duration.minutes(15),
+                statistic="Maximum",
+            ),
+            threshold=1,
+            evaluation_periods=1,
+            comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+            treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
+        )
+
+        # Send alarm notifications to the batch failure topic
+        alarm.add_alarm_action(cw_actions.SnsAction(self.batch_failure_topic))
+
+        return alarm

{workbench-0.8.233 → workbench-0.8.241}/aws_setup/workbench_dashboard_full/app.py

@@ -12,7 +12,7 @@ print(f"Account: {aws_account}")
 print(f"Region: {aws_region}")
 
 # When you want a different docker image change this line
-dashboard_image = "public.ecr.aws/m6i5k1r2/workbench_dashboard:v0_8_232_amd64"
+dashboard_image = "public.ecr.aws/m6i5k1r2/workbench_dashboard:v0_8_237_amd64"
 
 # Workbench Configuration
 try:

{workbench-0.8.233 → workbench-0.8.241}/aws_setup/workbench_dashboard_full/workbench_dashboard_full/workbench_dashboard_stack.py

@@ -1,5 +1,6 @@
 from typing import Optional, List
 from aws_cdk import (
+    Duration,
     Stack,
     StackProps,
     aws_ecs as ecs,
@@ -175,3 +176,6 @@ class WorkbenchDashboardStack(Stack):
 
         # Add our custom security group
         fargate_service.load_balancer.add_security_group(lb_security_group)
+
+        # Enable sticky sessions so users stay on the same container (preserves in-memory state)
+        fargate_service.target_group.enable_cookie_stickiness(Duration.hours(1))

{workbench-0.8.233 → workbench-0.8.241}/aws_setup/workbench_dashboard_lite/app.py

@@ -12,7 +12,7 @@ print(f"Account: {aws_account}")
 print(f"Region: {aws_region}")
 
 # When you want a different docker image change this line
-dashboard_image = "public.ecr.aws/m6i5k1r2/workbench_dashboard:v0_8_232_amd64"
+dashboard_image = "public.ecr.aws/m6i5k1r2/workbench_dashboard:v0_8_237_amd64"
 
 # Workbench Configuration
 try:

{workbench-0.8.233 → workbench-0.8.241}/aws_setup/workbench_dashboard_lite/workbench_dashboard_lite/workbench_dashboard_stack.py

@@ -1,5 +1,6 @@
 from typing import Optional, List
 from aws_cdk import (
+    Duration,
     Stack,
     StackProps,
     aws_ecs as ecs,
@@ -183,3 +184,6 @@ class WorkbenchDashboardStack(Stack):
 
         # Add our custom security group
         fargate_service.load_balancer.add_security_group(lb_security_group)
+
+        # Enable sticky sessions so users stay on the same container (preserves in-memory state)
+        fargate_service.target_group.enable_cookie_stickiness(Duration.hours(1))

{workbench-0.8.233 → workbench-0.8.241}/docs/advanced_models/index.md

@@ -100,15 +100,8 @@ model = feature_set.to_model(
 )
 ```
 
-## Getting Started
-
-1. Install Workbench: `pip install workbench`
-2. Connect to AWS®: [Setup Guide](../getting_started/index.md)
-3. Create your model using the examples above
-4. Deploy to an endpoint for inference
-
 !!! note "Examples"
-    All Workbench examples are in the repository under the `examples/` directory. For full code listings, visit [Workbench Examples](https://github.com/SuperCowPowers/workbench/blob/main/examples).
+    All Workbench model examples are in the repository under the `examples/models` directory. For full code listings, visit [Workbench Model Examples](https://github.com/SuperCowPowers/workbench/blob/main/examples/models).
 
 ---
 

workbench-0.8.241/docs/confidence/index.md

@@ -0,0 +1,111 @@
+# Confidence Scores in Workbench
+!!!tip inline end "Need Help?"
+    The SuperCowPowers team is happy to give any assistance needed when setting up AWS and Workbench. So please contact us at [workbench@supercowpowers.com](mailto:workbench@supercowpowers.com) or on chat us up on [Discord](https://discord.gg/WHAJuz8sw8)
+
+Workbench provides **confidence scores** for model predictions, giving users a measure of how certain the model is about each prediction. Higher confidence indicates the model is more certain, while lower confidence suggests greater uncertainty.
+
+## Overview
+
+| **Framework**  | **UQ Method**                          | **Confidence Source**           |
+|----------------|----------------------------------------|---------------------------------|
+| XGBoost        | MAPIE Conformalized Quantile Regression | 80% prediction interval width  |
+| PyTorch        | MAPIE Conformalized Quantile Regression | 80% prediction interval width  |
+| ChemProp       | Ensemble disagreement (5 models)       | Ensemble standard deviation     |
+
+All frameworks use the same confidence formula, but the uncertainty source differs.
+
+## Confidence Formula
+
+Confidence is computed using a simple exponential decay function:
+
+```
+confidence = exp(-uncertainty / median_uncertainty)
+```
+
+Where:
+
+- **uncertainty**: The model's uncertainty estimate for each prediction
+- **median_uncertainty**: The median uncertainty from the training/validation data
+
+
+## XGBoost and PyTorch: MAPIE CQR
+
+For XGBoost and PyTorch models, Workbench uses **MAPIE Conformalized Quantile Regression (CQR)** to generate prediction intervals with coverage guarantees.
+
+### How It Works
+
+1. **Train quantile models**: LightGBM models are trained to predict the 10th and 90th percentiles (80% confidence interval)
+2. **Conformalize**: MAPIE adjusts the intervals using a held-out calibration set to guarantee coverage
+3. **Compute interval width**: The uncertainty is the width of the 80% prediction interval (q_90 - q_10)
+4. **Calculate confidence**: Apply the exponential decay formula
+
+### Why 80% Confidence Interval?
+
+We use the 80% CI (q_10 to q_90) rather than other intervals because:
+
+- **68% CI** is too narrow and sensitive to noise
+- **95% CI** is too wide and less discriminating between samples
+- **80% CI** provides a good balance for ranking prediction reliability
+
+### Coverage Guarantees
+
+MAPIE's conformalization ensures that prediction intervals achieve their target coverage. For example, an 80% CI will contain approximately 80% of true values on the calibration set. This is a key advantage over simple quantile regression.
+
+## ChemProp: Ensemble Disagreement
+
+For ChemProp models, Workbench trains an **ensemble of 5 models** and uses their disagreement as the uncertainty measure.
+
+### How It Works
+
+1. **Train ensemble**: 5 ChemProp models are trained with different random seeds
+2. **Predict**: Each model makes a prediction for each sample
+3. **Compute disagreement**: The standard deviation across the 5 predictions is the uncertainty
+4. **Calibrate**: The standard deviation is empirically calibrated against actual errors
+5. **Calculate confidence**: Apply the exponential decay formula using calibrated std
+
+### Ensemble vs MAPIE
+
+| Aspect | MAPIE (XGBoost/PyTorch)   | Ensemble (ChemProp)        |
+|--------|---------------------------|----------------------------|
+| Coverage guarantee | Yes (conformal)           | No (empirical)             |
+| Computational cost | + 5 MAPIE models          | Just the 5 chemprop models |
+| Uncertainty type | Prediction interval width | Model disagreement         |
+
+Both approaches are valid and widely used in the ML community. MAPIE provides theoretical guarantees, while ensembles capture model uncertainty more directly.
+
+## Interpreting Confidence Scores
+
+### High Confidence (> 0.5)
+- Model is relatively certain about the prediction
+- Prediction interval is narrower than typical
+- Good candidates for prioritization
+
+### Medium Confidence (0.3 - 0.5)
+- Model has typical uncertainty
+- Prediction is likely reasonable but verify important decisions
+
+### Low Confidence (< 0.3)
+- Model is uncertain about the prediction
+- Prediction interval is wider than typical
+- May indicate out-of-distribution samples or difficult predictions
+
+## Metrics for Evaluating Confidence
+
+Workbench computes several metrics to evaluate how well confidence correlates with actual prediction quality:
+
+### confidence_to_error_corr
+Spearman correlation between confidence and absolute error. **Should be negative** (high confidence = low error). Target: < -0.5
+
+### interval_to_error_corr
+Spearman correlation between interval width and absolute error. **Should be positive** (wide intervals = high error). Target: > 0.5
+
+### Coverage Metrics
+For each confidence level (50%, 68%, 80%, 90%, 95%), the percentage of true values that fall within the prediction interval. Should match the target coverage.
+
+## Additional Resources
+
+<img align="right" src="../images/scp.png" width="180">
+
+Need help with confidence scores or uncertainty quantification? Want to develop a customized application tailored to your business needs?
+
+- Consulting Available: [SuperCowPowers LLC](https://www.supercowpowers.com)