workbench 0.8.484__tar.gz → 0.8.485__tar.gz

{workbench-0.8.484 → workbench-0.8.485}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: workbench
-Version: 0.8.484
+Version: 0.8.485
 Summary: Workbench: A Dashboard and Python API for creating and deploying AWS SageMaker Model Pipelines
 Author-email: SuperCowPowers LLC <support@supercowpowers.com>
 License: MIT License

{workbench-0.8.484 → workbench-0.8.485}/applications/aws_dashboard/Dockerfile

@@ -29,7 +29,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         -r /tmp/requirements.lock
 
 # Layer 2: Workbench only (changes often, ~20 MB)
-ARG WORKBENCH_VERSION=0.8.483
+ARG WORKBENCH_VERSION=0.8.484
 RUN pip install --no-deps "workbench==${WORKBENCH_VERSION}"
 
 # Copy app code and configuration (after pip installs to avoid cache invalidation)

{workbench-0.8.484 → workbench-0.8.485}/applications/aws_dashboard/requirements.lock

@@ -44,7 +44,7 @@ antlr4-python3-runtime==4.9.3
     # via
     #   -c ci/.uv-constraints.txt
     #   omegaconf
-anyio==4.13.0
+anyio==4.15.1
     # via
     #   -c ci/.uv-constraints.txt
     #   anthropic
@@ -144,7 +144,7 @@ contourpy==1.3.3
     # via
     #   -c ci/.uv-constraints.txt
     #   matplotlib
-cryptography==48.0.1
+cryptography==50.0.1
     # via
     #   -c ci/.uv-constraints.txt
     #   google-auth
@@ -905,7 +905,7 @@ tritonclient==2.69.0
     # via
     #   -c ci/.uv-constraints.txt
     #   sagemaker-serve
-typing-extensions==4.15.0
+typing-extensions==4.16.0
     # via
     #   -c ci/.uv-constraints.txt
     #   aiohttp

{workbench-0.8.484 → workbench-0.8.485}/aws_setup/workbench_dashboard/app.py

@@ -12,7 +12,7 @@ print(f"Account: {aws_account}")
 print(f"Region: {aws_region}")
 
 # When you want a different docker image change this line
-dashboard_image = "public.ecr.aws/m6i5k1r2/workbench_dashboard:v0_8_458_amd64"
+dashboard_image = "public.ecr.aws/m6i5k1r2/workbench_dashboard:v0_8_484_amd64"
 
 # Workbench Configuration
 try:

{workbench-0.8.484 → workbench-0.8.485}/aws_setup/workbench_dashboard/workbench_dashboard/workbench_dashboard_stack.py

@@ -179,13 +179,14 @@ class WorkbenchDashboardStack(Stack):
 
         # Build the load balancer explicitly: an ALB accepts only one subnet per AZ, and the
         # ECS pattern offers no way to scope the LB's subnets to the ones we were given.
+        # A public LB needs public subnets (route to an IGW); the given subnets serve Redis and the tasks.
         load_balancer = elbv2.ApplicationLoadBalancer(
             self,
             "WorkbenchLB",
             vpc=cluster.vpc,
             internet_facing=props.public,
             security_group=lb_security_group,
-            vpc_subnets=subnet_selection,
+            vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC) if props.public else subnet_selection,
         )
 
         # Adding LoadBalancer with Fargate Service

{workbench-0.8.484 → workbench-0.8.485}/docs/blogs/cyp_challenge.md

@@ -1,7 +1,7 @@
 # OpenADMET CYP Challenge — Working Notes
 
 !!! tip inline end "Current entry"
-    An average of four Chemprop models over the challenge data plus public CYP potency, stock hyperparameters, plus a per-isoform placement correction. **Macro ST-RAE 0.4378** on the live half of the test set.
+    An average of four Chemprop models over the challenge data plus public CYP potency, stock hyperparameters, plus a per-isoform calibration. **Macro ST-RAE 0.4378** on the live half of the test set.
 
 **Work in progress.** The [OpenADMET CYP Inhibition Blind Challenge](https://huggingface.co/spaces/openadmet/cyp-challenge) (Direct Inhibition track) is still open. The model changes between submissions, and the live leaderboard scores only half the test set. So these are just some early observations that may, or may not, translate well to the final leaderboard.
 
@@ -47,7 +47,7 @@ Four Chemprop D-MPNNs, averaged. All are SMILES-only with stock hyperparameters,
 | `2d6-isoform` | 6 | every CYP2D6 readout, nothing else |
 | `2d6-single` | 1 | CYP2D6 pIC50 alone |
 
-Only the four scored pIC50 heads are ever submitted; the rest exist to shape the representation. The specialists carry no other isoforms, so CYP1A2, CYP2C9 and CYP3A4 average two members and CYP2D6 averages four. Predictions are averaged, then placed.
+Only the four scored pIC50 heads are ever submitted; the rest exist to shape the representation. The specialists carry no other isoforms, so CYP1A2, CYP2C9 and CYP3A4 average two members and CYP2D6 averages four. Predictions are averaged, then calibrated.
 
 **Why multi-task.** The isoforms are correlated and the per-isoform data is small, 1,285–2,335 curves each. One encoder learning from all 6,525 measurements is a data-efficiency argument, not a claim that graph learning resolves activity cliffs; on activity cliffs descriptor models match or beat GNNs (van Tilborg et al.).
 
@@ -65,25 +65,27 @@ Clustered around hits is not the same as active: analogs of a potent compound ar
 
 The gap is estimable before submitting anything. The PubChem qHTS panel puts CYP2D6 inactivity near 65%; a set that inactive centres around pIC50 3.7, against the 4.69 a model trained on fitted curves predicts. Solved against the blind half afterwards, the true centre is 3.107. The blind population is also *wider* on all four isoforms. Squared-error models shrink toward the mean, and a label set built from successful fits is already narrower than the population it came from.
 
-## Placement
+## Calibration
+
+Calibration here is the affine kind — shift and scale — not interval coverage.
 
 Predictions carry two independent things. Their **order**, which is the model, and their **placement** on the axis, which is not. R² decomposes exactly:
 
 $$R^2 = 2\rho k - k^2 - b^2$$
 
-with ρ the Pearson correlation, `k = sd(pred)/sd(true)` the spread ratio, and `b` the mean offset in sd(true) units. Only ρ depends on the ordering; `k` and `b` come from an affine transform that touches no compound's rank. So **R² ≤ ρ²** is a hard ceiling, and a model far below its own ρ² is *mis-placed, not weak*.
+with ρ the Pearson correlation, `k = sd(pred)/sd(true)` the spread ratio, and `b` the mean offset in sd(true) units. Only ρ depends on the ordering; `k` and `b` come from an affine transform that touches no compound's rank. So **R² ≤ ρ²** is a hard ceiling, and a model far below its own ρ² is *mis-calibrated, not weak*. The decomposition is Murphy's (1988): the `k ≠ ρ` term is conditional bias, the `b` term unconditional bias.
 
 The optimum is **`k = ρ`, not `k = 1`**. Matching the spread of the truth is wrong: a model with ρ = 0.7 should be 70% as wide as reality, because shrinking toward the mean is the correct response to uncertainty. Raw predictions are narrower still.
 
-So: estimate the target population's centre and spread, then place each isoform there with spread `ρ·sd`.
+So: estimate the target population's centre and spread, then calibrate each isoform onto it with spread `ρ·sd`.
 
-<img src="../../images/cyp_calibration_applied.svg" alt="Four panels, one per isoform, each overlaying raw blind-set predictions, the same predictions after placement, and the blind population curve. CYP1A2, CYP2C9 and CYP3A4 shift and widen onto their population curves. CYP2D6 also moves down and widens but deliberately stops short, sitting above the population centre and narrower than it, because that is where the scored metric is optimised rather than R-squared." style="width: 100%; height: auto; display: block;">
+<img src="../../images/cyp_calibration_applied.svg" alt="Four panels, one per isoform, each overlaying raw blind-set predictions, the same predictions after calibration, and the blind population curve. CYP1A2, CYP2C9 and CYP3A4 shift and widen onto their population curves. CYP2D6 also moves down and widens but deliberately stops short, sitting above the population centre and narrower than it, because that is where the scored metric is optimised rather than R-squared." style="width: 100%; height: auto; display: block;">
 
 CYP2D6 shows the effect at full scale: raw predictions spike at 4.5 with sd 0.49 against a population centred at 3.1 with sd 1.60. It never calls a compound a non-inhibitor, and predicts into under a third of the actual range.
 
-Placement changes the score on identical weights and an identical ordering. Spearman and Kendall come back bit-identical under an affine transform, which doubles as the integrity check: if they move, the bug is in the submission pipeline.
+Calibration changes the score on identical weights and an identical ordering. Spearman and Kendall come back bit-identical under an affine transform, which doubles as the integrity check: if they move, the bug is in the submission pipeline.
 
-**The catch.** The placement that maximises R² does not minimise the scored metric. Soft-threshold RAE is zero anywhere inside a compound's credible interval, and low-activity compounds carry wide intervals, so predicting high is nearly free while predicting low is punished by the actives. Placing CYP2D6 on its true centre raises R² and *worsens* ST-RAE; there the two objectives point in opposite directions.
+**The catch.** The calibration that maximises R² does not minimise the scored metric. Soft-threshold RAE is zero anywhere inside a compound's credible interval, and low-activity compounds carry wide intervals, so predicting high is nearly free while predicting low is punished by the actives. Calibrating CYP2D6 onto its true centre raises R² and *worsens* ST-RAE; there the two objectives point in opposite directions.
 
 ## Open Problems
 
@@ -91,7 +93,7 @@ Placement changes the score on identical weights and an identical ordering. Spea
 
 **Both rulers are underpowered.** Repeated training runs of one configuration at different seeds disagree by more than candidate models do, so out-of-fold cross-validation cannot resolve the differences now being chased. The leaderboard is not much better: a Spearman's standard error is largest where the correlation is weakest, making CYP2D6 the worst-resolved isoform on both. Anything below the noise floor is treated as unmeasured rather than as a result.
 
-**Where the ST-RAE optimum sits.** Known by sampling placements against the board, not derived. A credible-interval width model would give it directly.
+**Where the ST-RAE optimum sits.** Known by sampling calibrations against the board, not derived. A credible-interval width model would give it directly.
 
 ## Reproducing This
 
@@ -111,7 +113,7 @@ python ml_pipelines/OpenADMET/cyp/cyp_chemprop_union.py --public-weight 0.30  #
 python ml_pipelines/OpenADMET/cyp/cyp_chemprop_2d6.py --scope isoform
 python ml_pipelines/OpenADMET/cyp/cyp_chemprop_2d6.py --scope single
 
-# Average their predictions over the 750 blinded compounds, then place them
+# Average their predictions over the 750 blinded compounds, then calibrate them
 python ml_pipelines/OpenADMET/cyp/scripts/cyp_ensemble_submit.py
 python ml_pipelines/OpenADMET/cyp/scripts/cyp_recalibrate.py --source outputs/<file> --strae
 ```
@@ -125,6 +127,7 @@ Submission files are checked with OpenADMET's own validator, vendored from their
 - [Building the OpenADMET Data Engine](https://openadmet.github.io/Octant_CYP_blog_post/?ref=openadmet.ghost.io) — Warneford-Thomson, Simpkins, Edgar and MacDermott-Opeskin on the assay platform behind this data: Echo-MS, 1536-well miniaturisation, and why well-level records are released alongside the fitted parameters
 - [A Weekend on the OpenADMET PXR Challenge](pxr_weekend_experiments.md) — our prior blind-challenge write-up, and where the HPO and 3D-descriptor priors come from
 - van Tilborg et al., *Exposing the Limitations of Molecular Machine Learning with Activity Cliffs*, J. Chem. Inf. Model. 2022
+- Murphy, *Skill Scores Based on the Mean Square Error and Their Relationships to the Correlation Coefficient*, [Mon. Weather Rev. 1988](https://journals.ametsoc.org/view/journals/mwre/116/12/1520-0493_1988_116_2417_ssbotm_2_0_co_2.xml) — the R² decomposition above, as an MSE skill score
 
 **Data sources**
 

{workbench-0.8.484 → workbench-0.8.485}/docs/blogs/uncertainty_quantification.md

@@ -42,10 +42,12 @@ The left and right panels are the cases ensemble std already handles: agreement
 Here's the whole path for a ChemProp model, from a batch of SMILES to the columns that land on every row.
 
 <figure style="margin: 20px auto; text-align: center;">
-<img src="../../images/uq_endpoint_flow.svg" alt="Deployed endpoint with UQ: SMILES in, 13 UQ columns out" style="width: 100%; height: auto;">
+<img src="../../images/uq_endpoint_flow.svg" alt="Deployed endpoint with UQ: ensemble and applicability domain feed five inputs to the RF UQ model" style="width: 100%; height: auto;">
 </figure>
 
-The ensemble runs first and produces two numbers per compound: `prediction` (the mean across folds) and `prediction_std` (the spread). Those two, plus the compound's SMILES, are everything the UQ model receives.
+The **RF UQ model takes five inputs from two sources.** The **ensemble** contributes `prediction` (the mean across folds) and `prediction_std` (the spread). The **applicability domain** contributes three more, computed from the compound's *k* nearest training neighbors by fingerprint similarity: `distance`, `target_std` and `pred_gap`.
+
+Neither source is sufficient alone. Ensemble spread says how much the model argues with itself; the applicability domain says whether the compound sits anywhere the training data can speak to. A random forest learns how the two together map to actual error.
 
 What comes back is 13 columns:
 
@@ -66,7 +68,7 @@ For a multi-target model every target gets its own prefixed set (`{target}_confi
 
 # v1: Conformalized Residual-Estimator
 
-v1 replaces "rank the ensemble std" with "**learn how the ensemble's signals map to actual error**, using the compound's neighborhood in chemical space." It's a small supervised model that predicts the magnitude of a prediction's error, conformalized to produce calibrated intervals. The approach is validated by the 2025 *J. Chem. Inf. Model.* study on UQ under data shift ([PMC12848971](https://pmc.ncbi.nlm.nih.gov/articles/PMC12848971/)), which found that error models built on `[prediction, ensemble variance, distance to training]` outperform standard UQ metrics across ADMET endpoints.
+v1 replaces "rank the ensemble std" with "**learn how the ensemble's signals map to actual error**, using the compound's **applicability domain**, its neighborhood in chemical space." It's a small supervised model that predicts the magnitude of a prediction's error, conformalized to produce calibrated intervals. The approach is validated by the 2025 *J. Chem. Inf. Model.* study on UQ under data shift ([PMC12848971](https://pmc.ncbi.nlm.nih.gov/articles/PMC12848971/)), which found that error models built on `[prediction, ensemble variance, distance to training]` outperform standard UQ metrics across ADMET endpoints.
 
 Almost all of the machinery is in the fitting, so that's what's worth walking through. Inference is the trivial half: five features, one forest, one multiply.
 
@@ -78,7 +80,7 @@ The two tracks are the part to notice. The **error model** is the forest that sh
 
 ## Step 1: Neighborhood Residual Features
 
-For each compound, v1 computes five scalar features that describe its local context in the training set (via a fingerprint `Proximity` backend). The first two are the ensemble signals; the last three come from the *k* nearest training neighbors (default k=10):
+For each compound, v1 computes five scalar features that describe its local context in the training set (via a fingerprint `Proximity` backend). The first two are the ensemble signals; the last three are the applicability-domain signals, computed from the *k* nearest training neighbors (default k=10):
 
 <table style="width: 100%;">
   <thead>

workbench-0.8.485/docs/images/uq_endpoint_flow.svg

@@ -0,0 +1,97 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 610" width="900" height="610">
+  <defs>
+    <linearGradient id="dataBox" x1="0" y1="0" x2="0" y2="1">
+      <stop offset="0%" stop-color="#5f6b7a"/>
+      <stop offset="100%" stop-color="#454f5c"/>
+    </linearGradient>
+    <linearGradient id="modelBox" x1="0" y1="0" x2="0" y2="1">
+      <stop offset="0%" stop-color="#3a86ff"/>
+      <stop offset="100%" stop-color="#2a66cc"/>
+    </linearGradient>
+    <linearGradient id="adBox" x1="0" y1="0" x2="0" y2="1">
+      <stop offset="0%" stop-color="#ff9f43"/>
+      <stop offset="100%" stop-color="#cc7a30"/>
+    </linearGradient>
+    <linearGradient id="uqBox" x1="0" y1="0" x2="0" y2="1">
+      <stop offset="0%" stop-color="#9b59b6"/>
+      <stop offset="100%" stop-color="#7d3f96"/>
+    </linearGradient>
+    <linearGradient id="outBox" x1="0" y1="0" x2="0" y2="1">
+      <stop offset="0%" stop-color="#00d4aa"/>
+      <stop offset="100%" stop-color="#00a080"/>
+    </linearGradient>
+    <marker id="ar" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
+      <path d="M2 1L8 5L2 9" fill="none" stroke="#8b95a3" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+  </defs>
+
+  <text x="450" y="34" text-anchor="middle" fill="#ccc" font-family="Helvetica, Arial" font-size="24" font-weight="bold">Deployed Endpoint with UQ</text>
+  <text x="450" y="58" text-anchor="middle" fill="#999" font-family="Helvetica, Arial" font-size="15">applicability domain and ensemble, together</text>
+
+  <rect x="340" y="82" width="220" height="54" rx="8" fill="url(#dataBox)"/>
+  <text x="450" y="106" text-anchor="middle" fill="white" font-family="Helvetica, Arial" font-size="17" font-weight="bold">SMILES batch</text>
+  <text x="450" y="126" text-anchor="middle" fill="#dde3ea" font-family="Helvetica, Arial" font-size="13">one row per compound</text>
+
+  <line x1="450" y1="136" x2="450" y2="152" stroke="#8b95a3" stroke-width="1.8"/>
+  <line x1="250" y1="152" x2="650" y2="152" stroke="#8b95a3" stroke-width="1.8"/>
+  <line x1="250" y1="152" x2="250" y2="170" stroke="#8b95a3" stroke-width="1.8" marker-end="url(#ar)"/>
+  <line x1="650" y1="152" x2="650" y2="170" stroke="#8b95a3" stroke-width="1.8" marker-end="url(#ar)"/>
+
+  <rect x="90" y="176" width="320" height="90" rx="8" fill="url(#modelBox)"/>
+  <text x="250" y="202" text-anchor="middle" fill="white" font-family="Helvetica, Arial" font-size="16" font-weight="bold">ChemProp 5-fold ensemble</text>
+  <rect x="98" y="216" width="56" height="32" rx="5" fill="#ffffff" opacity="0.18"/>
+  <text x="126" y="237" text-anchor="middle" fill="white" font-family="Helvetica, Arial" font-size="13">fold 1</text>
+  <rect x="160" y="216" width="56" height="32" rx="5" fill="#ffffff" opacity="0.18"/>
+  <text x="188" y="237" text-anchor="middle" fill="white" font-family="Helvetica, Arial" font-size="13">fold 2</text>
+  <rect x="222" y="216" width="56" height="32" rx="5" fill="#ffffff" opacity="0.18"/>
+  <text x="250" y="237" text-anchor="middle" fill="white" font-family="Helvetica, Arial" font-size="13">fold 3</text>
+  <rect x="284" y="216" width="56" height="32" rx="5" fill="#ffffff" opacity="0.18"/>
+  <text x="312" y="237" text-anchor="middle" fill="white" font-family="Helvetica, Arial" font-size="13">fold 4</text>
+  <rect x="346" y="216" width="56" height="32" rx="5" fill="#ffffff" opacity="0.18"/>
+  <text x="374" y="237" text-anchor="middle" fill="white" font-family="Helvetica, Arial" font-size="13">fold 5</text>
+
+  <rect x="490" y="176" width="320" height="90" rx="8" fill="url(#adBox)"/>
+  <text x="650" y="202" text-anchor="middle" fill="#3a2a12" font-family="Helvetica, Arial" font-size="16" font-weight="bold">Applicability Domain</text>
+  <text x="650" y="226" text-anchor="middle" fill="#4a3618" font-family="Helvetica, Arial" font-size="13">k = 10 nearest training neighbors</text>
+  <text x="650" y="248" text-anchor="middle" fill="#4a3618" font-family="Helvetica, Arial" font-size="13">by fingerprint similarity</text>
+
+  <line x1="250" y1="266" x2="250" y2="288" stroke="#8b95a3" stroke-width="1.8" marker-end="url(#ar)"/>
+  <line x1="650" y1="266" x2="650" y2="288" stroke="#8b95a3" stroke-width="1.8" marker-end="url(#ar)"/>
+
+  <rect x="130" y="294" width="640" height="136" rx="8" fill="url(#uqBox)"/>
+  <text x="450" y="324" text-anchor="middle" fill="white" font-family="Helvetica, Arial" font-size="20" font-weight="bold">RF UQ Model</text>
+
+  <text x="270" y="363" text-anchor="end" fill="#e8dcf0" font-family="Helvetica, Arial" font-size="13" font-weight="bold">Ensemble</text>
+  <rect x="286" y="344" width="170" height="28" rx="5" fill="#ffffff" opacity="0.18"/>
+  <text x="371" y="363" text-anchor="middle" fill="white" font-family="Helvetica, Arial" font-size="13">prediction</text>
+  <rect x="470" y="344" width="170" height="28" rx="5" fill="#ffffff" opacity="0.18"/>
+  <text x="555" y="363" text-anchor="middle" fill="white" font-family="Helvetica, Arial" font-size="13">prediction_std</text>
+
+  <text x="270" y="403" text-anchor="end" fill="#e8dcf0" font-family="Helvetica, Arial" font-size="13" font-weight="bold">KNN / AD</text>
+  <rect x="286" y="384" width="140" height="28" rx="5" fill="#ffffff" opacity="0.18"/>
+  <text x="356" y="403" text-anchor="middle" fill="white" font-family="Helvetica, Arial" font-size="13">distance</text>
+  <rect x="438" y="384" width="140" height="28" rx="5" fill="#ffffff" opacity="0.18"/>
+  <text x="508" y="403" text-anchor="middle" fill="white" font-family="Helvetica, Arial" font-size="13">target_std</text>
+  <rect x="590" y="384" width="140" height="28" rx="5" fill="#ffffff" opacity="0.18"/>
+  <text x="660" y="403" text-anchor="middle" fill="white" font-family="Helvetica, Arial" font-size="13">pred_gap</text>
+
+  <line x1="450" y1="430" x2="450" y2="454" stroke="#8b95a3" stroke-width="1.8" marker-end="url(#ar)"/>
+
+  <rect x="140" y="460" width="620" height="130" rx="8" fill="url(#outBox)" opacity="0.16" stroke="#00d4aa" stroke-width="1.5"/>
+  <text x="450" y="486" text-anchor="middle" fill="#00d4aa" font-family="Helvetica, Arial" font-size="17" font-weight="bold">13 columns added to every row</text>
+
+  <rect x="160" y="500" width="180" height="76" rx="6" fill="#00d4aa" opacity="0.12"/>
+  <text x="250" y="524" text-anchor="middle" fill="#ccc" font-family="Helvetica, Arial" font-size="15" font-weight="bold">confidence</text>
+  <text x="250" y="546" text-anchor="middle" fill="#9aa5b1" font-family="Helvetica, Arial" font-size="13">0 = don't trust it</text>
+  <text x="250" y="566" text-anchor="middle" fill="#9aa5b1" font-family="Helvetica, Arial" font-size="13">1 = trust it</text>
+
+  <rect x="360" y="500" width="180" height="76" rx="6" fill="#00d4aa" opacity="0.12"/>
+  <text x="450" y="524" text-anchor="middle" fill="#ccc" font-family="Helvetica, Arial" font-size="15" font-weight="bold">expected_residual</text>
+  <text x="450" y="546" text-anchor="middle" fill="#9aa5b1" font-family="Helvetica, Arial" font-size="13">predicted error size</text>
+  <text x="450" y="566" text-anchor="middle" fill="#9aa5b1" font-family="Helvetica, Arial" font-size="13">in target units</text>
+
+  <rect x="560" y="500" width="180" height="76" rx="6" fill="#00d4aa" opacity="0.12"/>
+  <text x="650" y="524" text-anchor="middle" fill="#ccc" font-family="Helvetica, Arial" font-size="15" font-weight="bold">q_025 … q_975</text>
+  <text x="650" y="546" text-anchor="middle" fill="#9aa5b1" font-family="Helvetica, Arial" font-size="13">11 interval bounds</text>
+  <text x="650" y="566" text-anchor="middle" fill="#9aa5b1" font-family="Helvetica, Arial" font-size="13">q_50 = prediction</text>
+</svg>

workbench-0.8.485/docs/presentations/pxr_repl_demo.md

@@ -0,0 +1,137 @@
+# Demo: REPL + Bosco on the PXR challenge
+
+A recorded walkthrough. The spine is the handoff — **REPL to explore, Bosco to build, REPL to
+inspect what it built** — repeated four times, on work that is real rather than staged.
+Edited for pace; say so on screen.
+
+**Why PXR.** One target (`pec50`), and a held-out set with real labels: OpenADMET revealed the
+253-compound phase-1 set, so every score is honest and nothing waits on a leaderboard. The
+contest is closed, so nothing here is a competitive claim.
+
+**Everything is ad-hoc.** Bosco creates the artifacts in-session. No scripts, no
+`pipelines.json`. That is the point being demonstrated: the artifacts are addressable the
+moment they exist, by name, from the REPL — no export step, no copy-paste, no notebook.
+
+**Naming.** Everything built on camera takes a `-demo` suffix. `openadmet_pxr_f1` and
+`pxr-reg-chemprop-phase1` already exist, and the latter carries the `pxr_phase1_test` capture
+the numbers below came from.
+
+---
+
+## 1. Public data to a local model — REPL → Bosco → REPL
+
+**REPL.** Pull the data and look at it. No credentials needed for either of these:
+
+```python
+train = PublicData().get("comp_chem/openadmet/pxr/training/main")            # 4,139
+test  = PublicData().get("comp_chem/openadmet/pxr/testing/phase1_unblinded") # 253
+```
+
+**Bosco.** Hand off the part that carries the traps: *"mark the split, compute 2D descriptors,
+build a local FeatureSet and a local XGBoost model holding out the phase-1 rows."*
+
+That one request covers `compute_descriptors`, picking a feature list that excludes
+`molecule_name` / `smiles` / `pec50` / its error and interval columns / the `split` marker, and
+wiring the holdout. Doing it by hand is where the mistakes live.
+
+**REPL.** The local model is on disk and addressable by name. Inspect it, check the feature
+list it chose, pull the out-of-fold predictions.
+
+2D only — `compute_descriptors` is RDKit plus Mordred and is fast. The 3D blocks need conformer
+generation and xTB, which is not a live-loop operation.
+
+Workbench models are a 5-fold CV ensemble, so one call yields both the out-of-fold predictions
+and the fold-averaged predictor.
+
+## 2. Residual plots — out of fold
+
+**REPL.** Interactive, you drive. Full cross-fold residuals set the visual baseline.
+
+## 3. Residual plots — phase-1 held out
+
+**REPL.** The same plot on the 253 revealed compounds. **This does not go the way the room
+expects, and that is the point.** Measured on the existing chemprop phase-1 model:
+
+| capture | n | MAE | R² | Pearson | Spearman | k = sd(pred)/sd(true) |
+|---|---|---|---|---|---|---|
+| `full_cross_fold` | 4,392 | 0.490 | **0.615** | 0.788 | 0.751 | 0.854 |
+| `pxr_phase1_test` | 253 | **0.472** | 0.527 | 0.752 | **0.797** | **0.630** |
+
+Held-out MAE and Spearman are *better* than cross-fold. R² is worse anyway. Let someone ask
+why — the answer is that R² fell while the ordering improved, and the whole cause is the spread
+ratio collapsing from 0.854 to 0.630. The model ranks the held-out set well and predicts into
+two thirds of its real range. In the plot that is systematic slope, not scatter.
+
+**Calibration, if you take it here.** Affine: shift and scale, no retraining, ranks untouched.
+The optimum is `k = ρ`, not `k = 1` — a model correlating at ρ should be ρ times as wide as
+reality, because shrinking toward the mean is the right response to uncertainty. Applied to the
+phase-1 predictions, R² goes 0.527 → 0.565, landing on its ρ² ceiling, Spearman bit-identical.
+
+Frame it as a deployment defect: a model predicting into two thirds of the range under-calls
+the extremes, and MAE never shows you that. Murphy (1988) if anyone wants the reference. Never
+mention leaderboards — this is measured against revealed labels, which is the whole reason it
+is defensible.
+
+## 4. Where the big residuals come from — REPL → Bosco → REPL
+
+**REPL.** Point at the outliers. Novel chemistry, or activity cliffs?
+
+**Bosco.** Ask it. The question already has columns attached — UQ v1 fits its error model on
+exactly `[prediction, prediction_std, knn_distance, knn_target_std, local_pred_gap]`.
+`knn_distance` is novel chemistry: nothing similar in training. `knn_target_std` and
+`local_pred_gap` are the cliff signal: close neighbours that disagree, or a prediction drifted
+from what its neighbourhood supports.
+
+**REPL.** Look at the compounds it separated, and their neighbours.
+
+This is the beat where Bosco is doing analysis rather than plumbing, which is a different and
+better argument for it.
+
+**Trap: do not plot confidence against error on `full_cross_fold`.** For a model built before
+2026-09-01 the cross-fold `confidence` came from the error model scoring its own training rows,
+so the correlation looks far better than it is. `pxr_phase1_test` is genuine held-out inference
+and is safe.
+
+## 5. The same thing as cloud artifacts — Bosco
+
+**Bosco.** *"Now build this for real: a FeatureSet and a chemprop model, `-demo` suffix, same
+253 rows held out, then deploy it."*
+
+Ad-hoc, in session. The FeatureSet build is quicker than it looks — `InferenceCache` keys the
+feature endpoint by SMILES, so the expensive 3D leg is already cached.
+
+Chemprop training is the natural cut point. Kick it, cut, return to a finished endpoint.
+
+Narrate over the cut: this is the artifact chain, not a notebook. DataSource → FeatureSet →
+Model → Endpoint, each registered, each carrying its own metrics and captures.
+
+## 6. The dashboard
+
+Where Workbench earns its keep, and the reason the cloud half exists in this demo. The model
+arrives as a tracked artifact with metrics, inference captures, UQ columns and plots attached —
+shareable, and still there next week. Nothing was written to a script to get it there.
+
+## 7. Chemprop vs local XGBoost — Bosco → REPL
+
+**Bosco.** *"Score both on the phase-1 rows and compare."*
+
+**REPL.** Read the table. Three things to say:
+
+- Quote the held-out numbers, not cross-fold. The table in §3 is why.
+- Quote Pearson or Spearman for a modelling difference. R² and MAE move with calibration, which
+  is not a model property — a badly calibrated model with better ordering is the better model,
+  and §3 is the proof.
+- It is not a clean architecture comparison. Local XGBoost reads locally-computed 2D
+  descriptors; cloud Chemprop reads SMILES and learns its own representation. Two variables,
+  not one. Say so rather than declaring a winner.
+
+---
+
+## Notes for the take
+
+- The local half needs no credentials at all. Worth saying out loud at §1.
+- Nothing here needs ST-RAE. The credible intervals are in `PublicData` under
+  `comp_chem/openadmet/pxr/testing/phase1_unblinded` if you want them.
+- 253 rows is small. A Pearson difference under ~0.05 between two models on that set is not a
+  result; say so rather than reading it as one.
+- The PXR contest is closed. No live-standing claims.

{workbench-0.8.484 → workbench-0.8.485}/lambda_layers/build_deploy.sh

@@ -26,7 +26,7 @@ WORKBENCH_ROOT="$(cd "$SCRIPT_DIR/.." &> /dev/null && pwd)"
 AWS_ACCOUNT_ID="507740646243"
 REGION_LIST=("us-east-1" "us-west-2")
 PYTHON_VERSION="3.12"
-WORKBENCH_VERSION=0.8.483   # keep in lockstep with the image's ARG WORKBENCH_VERSION
+WORKBENCH_VERSION=0.8.484   # keep in lockstep with the image's ARG WORKBENCH_VERSION
 LAMBDA_ARCH="manylinux2014_x86_64"
 
 GREEN='\033[0;32m'

{workbench-0.8.484 → workbench-0.8.485}/ml_pipelines/OpenADMET/cyp/cyp_chemprop_2d6.py

@@ -13,7 +13,7 @@ single-concentration readout is flat where the others are monotone, and the only
 emax carries potency signal. Several independent reasons to think it is not like the other
 three.
 
-Two scopes, both with an encoder that sees CYP2D6 and nothing else:
+Two scopes, both with an encoder that sees CYP2D6 and nothing else, `isoform` by default:
 
     --scope single    the scored target alone, 1,493 rows. Maximum isolation, minimum data.
     --scope isoform   every CYP2D6 readout we have, ~16k rows. Drops cross-isoform sharing
@@ -134,7 +134,7 @@ DEEP_BAND = 4.0
 parser = argparse.ArgumentParser(description=__doc__)
 parser.add_argument(
     "--scope",
-    required=True,
+    default="isoform",
     choices=["single", "isoform", "pooled", "tdi"],
     help="'single' trains on the scored target alone; 'isoform' adds every CYP2D6 readout; "
     "'pooled' puts offset-corrected public measurements into the scored column; "

{workbench-0.8.484 → workbench-0.8.485}/ml_pipelines/OpenADMET/cyp/cyp_chemprop_union.py

@@ -4,7 +4,8 @@ Many tasks on one shared encoder: the four scored pIC50 targets, the challenge a
 auxiliaries, and the public panels' potency and efficacy readouts. Only the scored four are
 ever submitted -- the rest exist to shape the representation.
 
-`--public-weight` is the experiment, and it moves more than its name suggests.
+`--public-weight` is the experiment, and it moves more than its name suggests. It defaults
+to 0.30, so a bare launcher run rebuilds `cyp-reg-chemprop-union-p30`.
 
 **A task's gradient share is its label count times its weight, not its share of the task
 list.** chemprop multiplies the per-element loss by `task_weights` and reduces by summing
@@ -121,7 +122,7 @@ parser = argparse.ArgumentParser(description=__doc__)
 parser.add_argument(
     "--public-weight",
     type=float,
-    required=True,
+    default=0.30,
     help="Per-head weight for the ChEMBL and Veith targets, as a multiple of mean(primary)",
 )
 parser.add_argument(

{workbench-0.8.484 → workbench-0.8.485}/ml_pipelines/OpenADMET/cyp/cyp_feature_sets.py

@@ -13,10 +13,12 @@ variant so a model can be compared across 3D sets with nothing else moving:
                              `_std`.
   - openadmet_cyp_tdi_<v>    TDI track. 6,145 compounds, `cyp3a4_is_tdi` and
                              `cyp2d6_is_tdi` as binary targets.
-Set `BUILD` to the variants this run should produce. A variant costs one xTB pass
-over the 6,145 TDI compounds — the regression set is a strict subset of them, and
-the InferenceCache is SMILES-keyed and S3-persisted, so both tracks and any rerun
-draw from that one pass.
+Each run builds one FeatureSet, selected by the pipeline mode `<track>_<variant>`
+(`inhibition_f1`, `inhibition_f2`, `tdi_f1`, `tdi_f2`), so each job in pipelines.json
+declares exactly its own public input, feature endpoint, and output. A variant costs
+one xTB pass over the 6,145 TDI compounds — the regression set is a strict subset of
+them, and the InferenceCache is SMILES-keyed and S3-persisted, so both tracks and any
+rerun draw from that one pass.
 
 Target names keep the challenge's own naming, snake_cased. That is not cosmetic:
 `scripts/cyp_compare.py` finds the credible interval by appending `_ci_lower` to the
@@ -27,12 +29,13 @@ they are NOT features. Every model script must pass an explicit `feature_list` -
 an auto-generated one includes them, and a model handed the bounds that bracket
 its own label scores near-perfectly and is worth nothing.
 
-Run once before the model scripts:  python cyp_feature_sets.py
+Run before the model scripts:  ml_pipeline_launcher cyp_feature_sets
 """
 
 import pandas as pd
 from workbench.api import DataSource, Endpoint, PublicData
 from workbench.api.inference_cache import InferenceCache
+from workbench.core.pipelines.pipeline_meta import PipelineMeta
 from workbench.utils.multi_task import validate_multi_task_data
 
 # Variant suffix -> the (Meta)Endpoint that produces its 2D+3D features. v1 is the
@@ -42,9 +45,6 @@ FEATURE_VARIANTS = {
     "f2": "smiles-to-2d-3d-v2",  # v2 3D (curated, xTB)
 }
 
-# Variants this run builds. Each is a multi-hour pass, so build them one at a time.
-BUILD = ["f2"]
-
 ISOFORMS = ["cyp3a4", "cyp2c9", "cyp2d6", "cyp1a2"]
 TARGETS = [f"{iso}_pic50_direct_inhibition" for iso in ISOFORMS]
 
@@ -59,39 +59,39 @@ def build_feature_set(fs_name: str, feature_endpoint: str, df: pd.DataFrame, tag
     print(f"Built '{fs_name}': {len(df)} rows, {n_feat} 2D+3D features; cache holds {cached.cache_size():,} SMILES")
 
 
-# --- Regression track: four sparse pIC50 targets with credible intervals -------------
-
-# Already one row per compound with NaN where an isoform has no fitted curve, which is
-# the multi-task shape -- no combining needed.
-inhibition = PublicData().get("comp_chem/openadmet/cyp/training/inhibition")
-validate_multi_task_data(inhibition, TARGETS, id_column="molecule_name")
+# Mode "<track>_<variant>" selects the one FeatureSet this run builds
+track, variant = PipelineMeta().mode.split("_")
 
-for variant in BUILD:
+if track == "inhibition":
+    # Regression track: four sparse pIC50 targets with credible intervals. Already one
+    # row per compound with NaN where an isoform has no fitted curve, which is the
+    # multi-task shape -- no combining needed.
+    inhibition = PublicData().get("comp_chem/openadmet/cyp/training/inhibition")
+    validate_multi_task_data(inhibition, TARGETS, id_column="molecule_name")
     build_feature_set(
         f"openadmet_cyp_{variant}",
         FEATURE_VARIANTS[variant],
         inhibition,
         tags=["openadmet_cyp", "multi_task", "activity"],
     )
-
-coverage = ", ".join(f"{t.split('_')[0]}={inhibition[t].notna().sum()}" for t in TARGETS)
-print(f"Regression targets: {coverage}")
-
-# --- TDI track: binary labels for the two scored isoforms ----------------------------
-
-# Labels only. `is_tdi` is derived from the shift between the direct and TDI arms, so
-# carrying either arm's pIC50 alongside the label would hand the model the answer.
-tdi = PublicData().get("comp_chem/openadmet/cyp/training/tdi")
-tdi = tdi[["molecule_name", "smiles", "cyp3a4_is_tdi", "cyp2d6_is_tdi"]].copy()
-
-for variant in BUILD:
+    coverage = ", ".join(f"{t.split('_')[0]}={inhibition[t].notna().sum()}" for t in TARGETS)
+    print(f"Regression targets: {coverage}")
+
+elif track == "tdi":
+    # TDI track: binary labels for the two scored isoforms. Labels only. `is_tdi` is
+    # derived from the shift between the direct and TDI arms, so carrying either arm's
+    # pIC50 alongside the label would hand the model the answer.
+    tdi = PublicData().get("comp_chem/openadmet/cyp/training/tdi")
+    tdi_targets = ["cyp3a4_is_tdi", "cyp2d6_is_tdi"]
+    tdi = tdi[["molecule_name", "smiles"] + tdi_targets].copy()
     build_feature_set(
         f"openadmet_cyp_tdi_{variant}",
         FEATURE_VARIANTS[variant],
         tdi,
         tags=["openadmet_cyp", "tdi", "classification"],
     )
+    positives = ", ".join(f"{c.split('_')[0]}={int(tdi[c].sum())}/{int(tdi[c].notna().sum())}" for c in tdi_targets)
+    print(f"TDI positives: {positives}")
 
-tdi_targets = ["cyp3a4_is_tdi", "cyp2d6_is_tdi"]
-positives = ", ".join(f"{c.split('_')[0]}={int(tdi[c].sum())}/{int(tdi[c].notna().sum())}" for c in tdi_targets)
-print(f"TDI positives: {positives}")
+else:
+    raise ValueError(f"Unknown track {track!r} in mode {PipelineMeta().mode!r}")

{workbench-0.8.484 → workbench-0.8.485}/ml_pipelines/OpenADMET/cyp/pipelines.json

@@ -1,13 +1,18 @@
 {
   "pipelines": {
     "openadmet_cyp": [
-      {"script": "cyp_feature_sets.py",
-       "inputs": ["public:comp_chem/openadmet/cyp/training/inhibition",
-                  "public:comp_chem/openadmet/cyp/training/tdi",
-                  "endpoint:smiles-to-2d-3d-v1",
-                  "endpoint:smiles-to-2d-3d-v2"],
-       "outputs": ["fs:openadmet_cyp_f1", "fs:openadmet_cyp_tdi_f1",
-                   "fs:openadmet_cyp_f2", "fs:openadmet_cyp_tdi_f2"]},
+      {"script": "cyp_feature_sets.py", "mode": "inhibition_f1",
+       "inputs": ["public:comp_chem/openadmet/cyp/training/inhibition", "endpoint:smiles-to-2d-3d-v1"],
+       "outputs": ["fs:openadmet_cyp_f1"]},
+      {"script": "cyp_feature_sets.py", "mode": "inhibition_f2",
+       "inputs": ["public:comp_chem/openadmet/cyp/training/inhibition", "endpoint:smiles-to-2d-3d-v2"],
+       "outputs": ["fs:openadmet_cyp_f2"]},
+      {"script": "cyp_feature_sets.py", "mode": "tdi_f1",
+       "inputs": ["public:comp_chem/openadmet/cyp/training/tdi", "endpoint:smiles-to-2d-3d-v1"],
+       "outputs": ["fs:openadmet_cyp_tdi_f1"]},
+      {"script": "cyp_feature_sets.py", "mode": "tdi_f2",
+       "inputs": ["public:comp_chem/openadmet/cyp/training/tdi", "endpoint:smiles-to-2d-3d-v2"],
+       "outputs": ["fs:openadmet_cyp_tdi_f2"]},
       {"script": "cyp_aux_features.py",
        "inputs": ["fs:openadmet_cyp_f1",
                   "public:comp_chem/openadmet/cyp/training/single_concentration"],