runbios-mcp 0.2.19 → 0.2.20-dev.286

package/dist/server.js

@@ -22,7 +22,7 @@ const RAW_FAILURE_TEXT_FIELDS = [
     "last_error",
     "stage_detail",
 ];
-import { LOOP_TOOL_PREFIX, resolveLaunchGates } from "./config.js";
+import { LOOP_TOOL_PREFIX, resolveLaunchGates, withoutGatedLoopIntrospection } from "./config.js";
 import { VERSION } from "./version.js";
 export { VERSION };
 /**
@@ -607,6 +607,58 @@ export function summarizeInferenceStatus(data) {
     }
     return shaped;
 }
+const TRAINING_PHASE_LABELS = {
+    dataset_download: "downloading datasets",
+    model_download: "downloading the model",
+    model_build: "building the model",
+    dataset_load: "reading the dataset",
+    preprocessing: "tokenizing samples",
+    validation_prep: "preparing validation batches",
+    model_load: "loading the model into GPU memory",
+    first_step: "first training step (compiling GPU kernels)",
+    training: "training",
+    checkpoint_upload: "saving a checkpoint",
+    artifact_commit: "finalizing results",
+};
+/**
+ * Summarize what a training job detail payload says about progress RIGHT NOW:
+ * the active step, its measured counts and rate, and the remaining time with
+ * what it covers. Everything is read from fields in the same response; a
+ * value that is absent stays null, never guessed.
+ */
+export function summarizeTrainingProgress(data) {
+    const payload = asRecord(data);
+    if (!payload)
+        return data;
+    const phases = Array.isArray(payload.phases) ? payload.phases.map(asRecord).filter(Boolean) : [];
+    if (phases.length === 0)
+        return data;
+    const active = phases.find((p) => p.state === "active") ?? null;
+    const num = (v) => (typeof v === "number" && Number.isFinite(v) ? v : null);
+    const phaseId = active ? nonEmptyString(active.phase) : undefined;
+    const eta = num(payload.eta_seconds);
+    const scope = nonEmptyString(payload.eta_scope);
+    const summary = {
+        current_phase: phaseId ?? null,
+        current_phase_label: phaseId ? (TRAINING_PHASE_LABELS[phaseId] ?? phaseId.replace(/_/g, " ")) : null,
+        done_units: active ? num(active.done_units) : null,
+        total_units: active ? num(active.total_units) : null,
+        units: active ? nonEmptyString(active.units) ?? null : null,
+        done_bytes: active ? num(active.done_bytes) : null,
+        total_bytes: active ? num(active.total_bytes) : null,
+        rate_per_second: active ? num(active.rate) : null,
+        phase_eta_seconds: active ? num(active.eta_seconds) : null,
+        phases_done: phases.filter((p) => p.state === "done").map((p) => p.phase),
+        phases_pending: phases.filter((p) => p.state === "pending").map((p) => p.phase),
+        eta_seconds: eta,
+        eta_confidence: nonEmptyString(payload.eta_confidence) ?? null,
+        eta_scope: scope ?? null,
+        note: "Derived by the MCP server from this same response. eta_scope 'until_training' means eta_seconds covers only the "
+            + "steps before training; training time is measured from the run's own steps once they run. 'measured' comes from "
+            + "this job's live rate, 'historical' from past runs of the same model and GPU. Null means not measured.",
+    };
+    return { ...payload, progress_summary: summary };
+}
 /* ─── Server setup ─── */
 export function createBiosMcpServer(client, hooks, deploymentCaps, 
 // Pre-launch surface gates (src/config.ts). While a gate is on, its creation
@@ -753,18 +805,18 @@ pipeline is deleted. Every other source is switched on with loop_set_capture.
 4. loop_add_signal / loop_list_signals - Feedback: accepted (good), rejected (bad), or edited with the corrected answer. Append-only. A correction is the most valuable feedback there is.
 5. loop_label_trace / loop_remove_label / loop_list_labels / loop_get_trace_pipelines - Tags. A pipeline trains on every conversation carrying any of its tags, and a conversation can carry several and feed several pipelines; loop_get_trace_pipelines says which pipelines one conversation feeds.
 6. loop_stats / loop_get_metrics_history - How much is recorded and reviewed, and how the attempts and the feedback moved over time.
-7. loop_list_models - The base models a pipeline can train, and the sample floors. The only list a pipeline's model may come from.
-8. loop_create_pipeline - One use case: a name, a model (optionally a challenger model), its tags, when to train (at least N new samples, and optionally a schedule) and how a new version is promoted. It switches on the source named by its own tag, so an app can send samples at once. SPEND CONSENT: saving it authorises attempts on the workspace's wallet whenever its data has grown enough; the platform chooses the GPUs and only the GPU time used is charged. Say so and wait for a yes.
-9. loop_list_pipelines / loop_get_pipeline / loop_update_pipeline / loop_delete_pipeline - Each pipeline's versions (winners only, v1, v2 ... numbers only go up), every attempt and how it ended, the live version, status and next_reason, and its settings.
+7. loop_list_models - The base models a pipeline can train, and the platform's fixed sample minimums (floors). The only list a pipeline's model may come from.
+8. loop_create_pipeline - One use case: a name, a model (optionally a challenger model), its tags, when to train (optionally a schedule; the sample minimums are fixed by the platform) and how a new version is promoted. It switches on the source named by its own tag, so an app can send samples at once. SPEND CONSENT: saving it authorises attempts on the workspace's wallet whenever its data has grown enough; the platform chooses the GPUs and only the GPU time used is charged. Say so and wait for a yes.
+9. loop_list_pipelines / loop_get_pipeline / loop_update_pipeline / loop_delete_pipeline - Each pipeline's versions (winners only, v1, v2 ... numbers only go up), every attempt and how it ended, the live version and whether its deployment answers (live_deployment), status and next_reason, and its settings. Deleting a pipeline stops its live version's deployment (unless another pipeline or model name answers from it too) and deletes its history.
 10. loop_run_pipeline / loop_import_into_pipeline - Train now (skips the schedule, never the minimum), and bring a file in as this pipeline's samples.
-11. loop_create_benchmark / loop_list_benchmarks / loop_get_benchmark / loop_list_benchmark_items / loop_get_benchmark_history / loop_retire_benchmark / loop_get_benchmark_run - Fixed sets of conversations every attempt is replayed on, so "is it getting better" has a yardstick that does not move.
+11. loop_create_benchmark / loop_list_benchmarks / loop_get_benchmark / loop_list_benchmark_items / loop_get_benchmark_history / loop_retire_benchmark / loop_get_benchmark_run / loop_list_benchmark_run_items - Fixed sets of conversations every attempt is replayed on, so "is it getting better" has a yardstick that does not move, and each replay's report: its scores, how many conversations each model passed (score 0.70 or more), and every conversation behind them.
 12. loop_list_pipeline_benchmarks / loop_attach_pipeline_benchmark / loop_move_pipeline_benchmark / loop_remove_pipeline_benchmark / loop_compare_versions - A pipeline's benchmarks in priority order (1 is the primary), and its versions side by side (v1 against v5).
 13. loop_list_training_runs / loop_get_training_run - Every attempt: its timeline, and available_actions — what this reader may actually do next, rather than what the state looks like it allows.
-14. loop_get_evaluation / loop_list_evaluation_items - The comparison behind a verdict: the win rate and the per-dimension means on the held-back samples, and the paired conversations themselves. Read the warnings out, not just the verdict.
+14. loop_get_evaluation / loop_list_evaluation_items - The comparison behind a verdict: the win rate, the pass rates (the share of conversations each model passed: score 0.70 or more) and the per-dimension means on the held-back samples, and the paired conversations themselves, with the judge's reasons. Read the warnings out, not just the verdict.
 15. loop_promote_training_run / loop_reject_training_run / loop_rollback_training_run / loop_cancel_training_run - The decision. Promotion makes an attempt the live version and changes what real customers get; it is undoable for 30 days.
 16. set_inference_alias / list_inference_aliases / delete_inference_alias - The re-pointable public handles a promotion writes: a name customers call that can be moved to different weights without them changing anything.
 
-END TO END: capture what a model was asked and answered (or import it), say whether each answer was good, bad or corrected, and tag it; create ONE pipeline per use case. When at least min_samples new samples carry its tags (and its schedule is due, if it has one) the platform trains an attempt, compares it with the live version on held-back samples it never trains on and on the pipeline's benchmarks, and promotes it only when it is better by the promotion policy — so each version beats the one before it. Everything before the pipeline costs nothing; saving a pipeline is where money starts, which is why it is spend consent. Two fields carry the state a user actually asks about and BOTH must be read back in plain words rather than summarised as 'fine': status says whether the platform is working or the user has to act, and next_reason says what a quiet pipeline is waiting for, because a pipeline waiting for samples looks identical to a broken one.`;
+END TO END: capture what a model was asked and answered (or import it), say whether each answer was good, bad or corrected, and tag it; create ONE pipeline per use case. When enough new samples carry its tags to meet the platform's fixed minimum (loop_list_models floors; the pipeline's minimums say how many are still needed) and its schedule is due, if it has one, the platform trains an attempt, compares it with the live version on held-back samples it never trains on and on the pipeline's benchmarks, and promotes it only when it is better by the promotion policy — so each version beats the one before it. Everything before the pipeline costs nothing; saving a pipeline is where money starts, which is why it is spend consent. Two fields carry the state a user actually asks about and BOTH must be read back in plain words rather than summarised as 'fine': status says whether the platform is working or the user has to act, and next_reason says what a quiet pipeline is waiting for, because a pipeline waiting for samples looks identical to a broken one.`;
     const server = {
         tool(name, description, paramsSchema, cb) {
             if (allowedTools && !allowedTools.has(name))
@@ -1541,7 +1593,9 @@ create (1-5 choices), and the queue itself stays opt-in.`,
     /* ══════════════════════════════════════════════════════════════════════════ */
     server.tool("introspect_api_key", "Discover what this API key can do. Returns the key's permissions (scopes), the org and workspace it's bound to, allowed MCP tools and SDK methods, rate limits, and expiration. Call this first to understand your access level and available capabilities.", {}, async () => {
         const data = await client.api("/api/api-keys/introspect");
-        return json(data);
+        // Where the loop is gated, IAM's loop tools and descriptions are removed:
+        // the answer must not advertise what tools/list hides (src/config.ts).
+        return json(withoutGatedLoopIntrospection(data, launchGates));
     });
     /* ══════════════════════════════════════════════════════════════════════════ */
     /*  TOOL: get_wallet_balance                                                */
@@ -2554,7 +2608,7 @@ create (1-5 choices), and the queue itself stays opt-in.`,
     /* ══════════════════════════════════════════════════════════════════════════ */
     /*  TOOL: get_training_status                                               */
     /* ══════════════════════════════════════════════════════════════════════════ */
-    server.tool("get_training_status", "Get detailed status for a specific training job: progress percentage, current step/total steps, current loss, learning rate, elapsed time, estimated time remaining, GPU utilization, and job configuration. Status 'securing' means a create is still booking a pod (poll until 'booked'); 'booked' means the GPU is secured and the job is starting. 'interrupted' is a self-heal rest state after a started job loses its pod — billing is already stopped and it is distinct from 'failed'; call resume_training_job to continue from the last checkpoint.", {
+    server.tool("get_training_status", "Get detailed status for a specific training job: progress percentage, current step/total steps, current loss, learning rate, elapsed time, estimated time remaining, GPU utilization, and job configuration. Status 'securing' means a create is still booking a pod (poll until 'booked'); 'booked' means the GPU is secured and the job is starting. 'interrupted' is a self-heal rest state after a started job loses its pod — billing is already stopped and it is distinct from 'failed'; call resume_training_job to continue from the last checkpoint. progress_summary names the step running now (downloading, tokenizing samples, loading into GPU memory, first-step compile, training...) with its measured counts, rate and remaining time; eta_scope 'until_training' means eta_seconds covers only the steps before training, 'run' the rest of the run.", {
         job_id: z.string().describe("Training job ID (e.g., 'job_abc123')"),
     }, async ({ job_id }) => {
         validateId(job_id, "job_id");
@@ -2563,7 +2617,7 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         // call to the machine, so it carries the request URL and with it the
         // supplier's hostname. Redact the raw-text fields only; model ids, config
         // and every other value pass through untouched.
-        return json(redactFreeTextFields(data, RAW_FAILURE_TEXT_FIELDS));
+        return json(summarizeTrainingProgress(redactFreeTextFields(data, RAW_FAILURE_TEXT_FIELDS)));
     });
     /* ══════════════════════════════════════════════════════════════════════════ */
     /*  TOOL: get_training_metrics                                              */
@@ -2918,7 +2972,7 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         });
         return json(data);
     });
-    server.tool("loop_import_rows", "Bring data the user ALREADY HAS into the loop — an export from another provider, a spreadsheet of past answers, a set of preference pairs. This does NOT create a training set: it creates conversations, in the same place captured ones live, subject to the same review, tags and pipelines. A row becomes trainable when something says it is good, never because it arrived in a file. Each row is read for what it is: a prompt/chosen/rejected triple becomes a preference pair; an answer plus a yes-or-no becomes a thumbs verdict; a question and an answer waits for review; a question with no answer waits for an answer; a paragraph of prose is refused, because it is not a conversation. A verdict that arrives with the file is kept, but recorded as having come from the user's earlier process rather than from a reviewer here. WHAT COMES BACK, and what to tell the user: imported is what this call created and trace_ids names the conversations from the file that are now in the loop; already_present is rows an earlier import of the same file already had, so they were not stored twice; by_shape counts the shapes this call CREATED, not what was merely recognised and not rows that were already here, so a file that was silently the wrong shape shows up there rather than merely importing fewer rows than expected, and reviewed and needs_review likewise count only what this call wrote; refused with refused_why is rows whose shape could not be read, which the user fixes by changing the file; not_saved with not_saved_rows is rows that were read and understood and then could not be stored, which the file cannot fix; verdicts_not_saved with verdicts_not_saved_rows is rows whose conversation stored and whose VERDICT did not, so those conversations are in the loop waiting for review rather than reviewed. A call with any not_saved or verdicts_not_saved rows fails with code IMPORT_INCOMPLETE and carries all of the same counts. HOW TO RETRY SAFELY: every response carries import_id, the token that call was filed under. Send the SAME rows again with that import_id and anything already imported comes back under already_present instead of being stored a second time, while a verdict that failed to write is attempted again. Do NOT retry without repeating import_id: a call that repeats no token is its own import and stores everything again, deliberately, because two calls carrying the same rows are as likely to be two pages of one export as one call sent twice. SENDING MORE THAN 5000 ROWS: split the file and either give every row its own row_ids entry, after which chunking and ordering stop mattering, or send one import_id with the row_offset each page starts at. Without row_ids, do not send back only the rows not_saved_rows names: a shorter list moves every row after the gap and imports it again.", {
+    server.tool("loop_import_rows", "Bring data the user ALREADY HAS into the loop — an export from another provider, a spreadsheet of past answers, a set of preference pairs. This does NOT create a training set: it creates conversations, in the same place captured ones live, subject to the same review, tags and pipelines. A row becomes trainable when something says it is good, never because it arrived in a file. Each row is read for what it is: a prompt/chosen/rejected triple becomes a correction (chosen is learned from); an answer plus a yes-or-no becomes a thumbs verdict; a question and an answer waits for review; a question with no answer waits for an answer; a paragraph of prose is refused, because it is not a conversation. A verdict that arrives with the file is kept, but recorded as having come from the user's earlier process rather than from a reviewer here. WHAT COMES BACK, and what to tell the user: imported is what this call created and trace_ids names the conversations from the file that are now in the loop; already_present is rows an earlier import of the same file already had, so they were not stored twice; by_shape counts the shapes this call CREATED, not what was merely recognised and not rows that were already here, so a file that was silently the wrong shape shows up there rather than merely importing fewer rows than expected, and reviewed and needs_review likewise count only what this call wrote; refused with refused_why is rows whose shape could not be read, which the user fixes by changing the file; not_saved with not_saved_rows is rows that were read and understood and then could not be stored, which the file cannot fix; verdicts_not_saved with verdicts_not_saved_rows is rows whose conversation stored and whose VERDICT did not, so those conversations are in the loop waiting for review rather than reviewed. A call with any not_saved or verdicts_not_saved rows fails with code IMPORT_INCOMPLETE and carries all of the same counts. HOW TO RETRY SAFELY: every response carries import_id, the token that call was filed under. Send the SAME rows again with that import_id and anything already imported comes back under already_present instead of being stored a second time, while a verdict that failed to write is attempted again. Do NOT retry without repeating import_id: a call that repeats no token is its own import and stores everything again, deliberately, because two calls carrying the same rows are as likely to be two pages of one export as one call sent twice. SENDING MORE THAN 5000 ROWS: split the file and either give every row its own row_ids entry, after which chunking and ordering stop mattering, or send one import_id with the row_offset each page starts at. Without row_ids, do not send back only the rows not_saved_rows names: a shorter list moves every row after the gap and imports it again.", {
         source: z.string().describe("Where this came from, e.g. 'zendesk-2026' or 'gpt4-history'. Required, and becomes the source every imported conversation is filed under — do not use a generic word like 'import', because that produces a corpus nobody can slice later."),
         rows: z.array(z.record(z.string(), z.any())).describe("The rows themselves, already parsed. At most 5000 per call: the call is synchronous. Pass each row in whatever shape it already has rather than reshaping it — the shape is how the server decides whether it carries a verdict."),
         model: z.string().optional().describe("Which model produced these answers, when the rows do not say per-row"),
@@ -2934,7 +2988,7 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         });
         return json(data);
     });
-    server.tool("loop_list_traces", "List recorded conversations (samples) for this workspace, newest first. To read a pipeline's data by feedback, pass pipeline with signal ('good', 'bad' or 'corrected': the verdict that decides the sample) or unsignalled true (what still needs review). A bad sample is never trained on. from/to narrow by when a conversation was recorded.", {
+    server.tool("loop_list_traces", "List recorded conversations (samples) for this workspace, newest first. To read a pipeline's data by feedback, pass pipeline with signal ('good', 'bad' or 'corrected': the verdict that decides the sample) or unsignalled true (what still needs review). A bad sample is never trained on. from/to narrow by when a conversation was recorded. A conversation kept in the workspace's own storage carries payload_stored_externally; when its text could not be read just now it also carries payload_unavailable, with messages null and completion empty. That conversation is NOT empty: say its text could not be read right now, do not review it, and read it again later.", {
         source: z.string().optional().describe("Only conversations from this source"),
         model: z.string().optional().describe("Only what THIS model answered, e.g. 'kimi-k3'. The filter distillation is made of — every other filter selects by what was asked or what somebody said about the answer, never by who produced it."),
         origin: z.enum(["captured", "imported"]).optional().describe("captured = the workspace's own model behaviour; imported = a file somebody brought in. Pooling them makes 'train only on what we actually served' unanswerable."),
@@ -2975,28 +3029,25 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         });
         return json(data);
     });
-    server.tool("loop_get_trace", "Read one recorded conversation in full, with every verdict anyone has recorded on it. Anything that looked like a credential or a personal detail was removed before it was stored, and redaction_report says what.", { trace_id: z.string().describe("The trace id returned by loop_capture_trace or loop_list_traces") }, async ({ trace_id }) => {
+    server.tool("loop_get_trace", "Read one recorded conversation in full, with every verdict and tag anyone has recorded on it. Anything that looked like a credential or a personal detail was removed before it was stored, and redaction_report says what. payload_unavailable true (messages null, completion empty) means its text is kept in the workspace's own storage and could not be read just now, not that it is empty: say so, do not review it, and read it again later.", { trace_id: z.string().describe("The trace id returned by loop_capture_trace or loop_list_traces") }, async ({ trace_id }) => {
         const data = await client.api(`/api/loop/traces/${encodeURIComponent(trace_id)}`);
         return json(data);
     });
     server.tool("loop_delete_trace", "Delete one recorded conversation, with its feedback and tags, for good. It stops counting toward every pipeline that selected it. A version already trained on it keeps what it learned. Confirm with the user first: this cannot be undone.", { trace_id: z.string() }, async ({ trace_id }) => json(await client.api(`/api/loop/traces/${encodeURIComponent(trace_id)}`, { method: "DELETE" })));
-    server.tool("loop_add_signal", "Record a verdict on a conversation: was the answer right? Feedback is APPEND-ONLY — a second verdict does not replace the first, because two reviewers disagreeing is information worth keeping.\n\nWHICH VERDICT TO USE, because they train different things:\n• accepted (a thumbs up) — the answer was good. It becomes an example to learn from.\n• rejected (a thumbs down) — it was wrong and you have nothing better. This REMOVES the answer from training rather than teaching anything; prefer edited whenever you know the better answer.\n• edited — the model was WRONG and here is what it should have said. The most valuable single action: it produces both halves of a preference pair, so the model learns your answer and learns to avoid its own.\n• gold — the REFERENCE answer for this question, recorded whatever the model happened to say. Unlike edited it asserts nothing about the model, so if it matches what was said no preference is invented — but it still trains SFT, and it stands in as the GRPO reference when no separate ground_truth was given. It is the most reusable feedback there is.\n• scored — a number, with ground_truth when the answer is checkable.", {
+    server.tool("loop_add_signal", "Record a verdict on a conversation: was the answer right? Feedback is APPEND-ONLY — a second verdict does not replace the first, because two reviewers disagreeing is information worth keeping.\n\nWHAT EACH VERDICT DOES (every pipeline fine-tunes on examples):\n• accepted (a thumbs up) — the answer was good. It becomes an example as it stands.\n• rejected (a thumbs down) — it was wrong and you have nothing better. It is LEFT OUT of training and nothing is learned from it; prefer edited whenever you know the better answer.\n• edited — the model was WRONG and here is what it should have said. The most valuable single action: your answer is trained on in place of the model's. Nothing is learned from the answer it replaced.\n• gold — the reference answer for this question, recorded whatever the model happened to say. It trains exactly like edited; it only makes no claim that the model was wrong.", {
         trace_id: z.string(),
-        verdict: z.enum(["accepted", "rejected", "edited", "scored", "gold"]).describe("accepted = thumbs up; rejected = thumbs down with nothing better to offer; edited = the model was wrong and you wrote the better answer; gold = the reference answer for this question regardless of what the model said; scored = a number or checkable fact"),
-        source: z.enum(["human", "verifier", "judge", "behavioural"]).optional().describe("Who judged. A human outranks a verifier, a verifier outranks a judge, a judge outranks a behavioural hint. Default human."),
+        verdict: z.enum(["accepted", "rejected", "edited", "gold"]).describe("accepted = thumbs up; rejected = thumbs down with nothing better to offer; edited = the model was wrong and you wrote the better answer; gold = the reference answer for this question regardless of what the model said"),
         correction: z.string().optional().describe("REQUIRED when verdict is 'edited' (the answer it should have given) or 'gold' (the reference answer for this question)"),
-        score: z.number().optional().describe("REQUIRED when verdict is 'scored'"),
-        ground_truth: z.string().optional().describe("The value or fact the answer can be checked against"),
         reason: z.string().optional().describe("Why, in your own words"),
         author: z.string().optional(),
         labels: z.array(z.string()).optional().describe("Which pipeline this feedback feeds, named HERE rather than in a second call: a pipeline trains on every conversation carrying any of its tags, so a tag is the pipeline a conversation goes down. Sent with the verdict it is written in one transaction: either both land or neither does. A separate tagging call is the one that gets skipped, and what it leaves is a reviewed conversation in no pipeline — counted in every 'reviewed' total and selected by nothing."),
-        attributes: z.record(z.string(), z.string()).optional().describe("The same, by named dimension: {\"category\": \"billing\"}. Dimensions AND together when a set is built, which is what lets one corpus become a different dataset per task."),
+        attributes: z.record(z.string(), z.string()).optional().describe("The same, by named dimension: {\"category\": \"billing\"}."),
         parent: z.string().optional().describe("The master group the bare labels belong to, so selecting the group picks them up without anybody maintaining a list. Leave it out and a label the conversation already carries keeps the group it has; send an empty string to take the labels out of their group."),
-    }, async ({ trace_id, verdict, source, correction, score, ground_truth, reason, author, labels, attributes, parent }) => {
+    }, async ({ trace_id, verdict, correction, reason, author, labels, attributes, parent }) => {
         const data = await client.api(`/api/loop/traces/${encodeURIComponent(trace_id)}/signals`, {
             method: "POST",
             body: {
-                verdict, source: source ?? "human", correction, score, ground_truth, reason, author,
+                verdict, source: "human", correction, reason, author,
                 labels, attributes, parent,
             },
         });
@@ -3063,14 +3114,17 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         model: pipelineModelSchema.optional().describe("The base model to improve"),
         challenger_model: pipelineModelSchema.nullable().optional().describe("Optional second model: after each attempt it is trained on the same data, and it too must beat the live version to become one. null removes it."),
         tags: z.array(z.string()).optional().describe("More tags whose samples this pipeline trains on, beside its own. On an edit the list replaces the previous one; the own tag stays."),
+        // STRICT: the sample minimums are fixed by the platform, and the service
+        // refuses a train_when.min_samples by name. zod would otherwise strip the
+        // key in silence, the save would succeed, and the agent would tell the user
+        // it had set a minimum nothing set.
         train_when: z.object({
-            min_samples: z.number().int().min(1).optional().describe("Train when at least this many NEW samples exist. The default and the floor are loop_list_models' floors.new_samples; below it is refused MIN_SAMPLES_TOO_LOW. The first version also needs floors.first_version_samples."),
             schedule: z.object({
                 every: z.enum(["day", "week"]),
                 weekday: z.number().int().min(0).max(6).optional().describe("0 is Sunday. Required for 'week'; leave it out for 'day'."),
                 hour_utc: z.number().int().min(0).max(23).describe("The UTC hour"),
-            }).nullable().optional().describe("Also wait for this time. Training starts only when BOTH the minimum and the schedule are met. null trains as soon as the minimum is met."),
-        }).optional(),
+            }).nullable().optional().describe("Also wait for this time. Training starts only when BOTH the platform's fixed sample minimum and the schedule are met. null trains as soon as the minimum is met."),
+        }).strict().optional().describe("When to train, beyond the platform's fixed sample minimum: only a schedule. There is no min_samples: the minimums are fixed (loop_list_models floors) and cannot be set."),
         promotion: z.object({
             policy: z.enum(["holdout", "all", "primary", "k_of_n", "weighted", "manual"]).describe("holdout (default): better on the held-back samples. all / primary / k_of_n / weighted: by the pipeline's benchmarks. manual: never promoted without a person."),
             k: z.number().int().min(1).optional().describe("For k_of_n only: how many measurements must improve"),
@@ -3078,8 +3132,8 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         max_versions: z.number().int().min(1).max(100).nullable().optional().describe("How many versions it may make, 1 to 100; null (the default) is no limit. A version is an attempt that became the live model; attempts that lost are not counted."),
         enabled: z.boolean().optional().describe("false saves it switched off"),
     };
-    server.tool("loop_list_models", "The base models a pipeline can train — the ONLY models loop_create_pipeline accepts — recommended first, each with its name, author, family and size, and floors: first_version_samples (what a new pipeline's first attempt needs) and new_samples (the default and smallest train_when.min_samples). Offer the user a choice from this list; never type a model id from memory.", {}, async () => json(await client.api("/api/loop/models")));
-    server.tool("loop_create_pipeline", "Create a pipeline: one use case, trained again whenever enough new samples carrying its tags exist. SPEND CONSENT: saving it authorises the platform to train attempts on the workspace's wallet with nobody present — whenever train_when is met it trains on GPUs the platform chooses, compares the attempt with the live version, and promotes it by the promotion policy. Only the GPU time actually used is charged. Say that in plain words and WAIT FOR AN EXPLICIT YES before calling. Take model.id from loop_list_models, never from memory. Nothing is saved when it is refused. 409 PIPELINE_NAME_TAKEN or OWN_TAG_TAKEN: choose another name. 422 MIN_SAMPLES_TOO_LOW, NO_GPU_FITS, MODEL_NOT_TRAINABLE or PROMOTION_*: the message names the fix. 422 NO_JUDGE_MODEL: no listed model can score the attempts; tell the user rather than retrying. 422 MODEL_REVISION_UNAVAILABLE: the model could not be pinned because the training service did not answer; try again shortly. 422 METHOD_NOT_ENABLED or CEILING_TOO_LOW: the platform cannot train this model now and nothing in the request changes that; tell the user. 409 TRAINING_NOT_ENABLED: training is in beta and the organisation is not enabled yet — tell the user the platform team enables it, and do not retry. 503 MODELS_UNAVAILABLE, GPU_OPTIONS_UNAVAILABLE or AGENT_COULD_NOT_START: a service the save needs did not answer just then; try again in a minute.", pipelineInputShape, async (input) => json(await client.api("/api/loop/pipelines", { method: "POST", body: input })));
+    server.tool("loop_list_models", "The base models a pipeline can train — the ONLY models loop_create_pipeline accepts — recommended first, each with its name, author, family and size, and floors, the platform's fixed sample minimums (read-only; no pipeline sets them): first_version_samples (what a new pipeline's first attempt needs, the held-back samples included), and new_samples and new_samples_percent (each later attempt waits for new_samples new samples, or new_samples_percent percent of the samples the last trained attempt trained on when that is more). Offer the user a choice from this list; never type a model id from memory.", {}, async () => json(await client.api("/api/loop/models")));
+    server.tool("loop_create_pipeline", "Create a pipeline: one use case, trained again whenever enough new samples carrying its tags exist. SPEND CONSENT: saving it authorises the platform to train attempts on the workspace's wallet with nobody present — whenever train_when is met it trains on GPUs the platform chooses, compares the attempt with the live version, and promotes it by the promotion policy. Only the GPU time actually used is charged. Say that in plain words and WAIT FOR AN EXPLICIT YES before calling. Take model.id from loop_list_models, never from memory. Nothing is saved when it is refused. 409 PIPELINE_NAME_TAKEN or OWN_TAG_TAKEN: choose another name. 422 NO_GPU_FITS, MODEL_NOT_TRAINABLE or PROMOTION_*: the message names the fix. 422 NO_JUDGE_MODEL: no listed model can score the attempts; tell the user rather than retrying. 422 MODEL_REVISION_UNAVAILABLE: the model could not be pinned because the training service did not answer; try again shortly. 422 METHOD_NOT_ENABLED or CEILING_TOO_LOW: the platform cannot train this model now and nothing in the request changes that; tell the user. 409 TRAINING_NOT_ENABLED: training is in beta and the organisation is not enabled yet — tell the user the platform team enables it, and do not retry. 503 MODELS_UNAVAILABLE, GPU_OPTIONS_UNAVAILABLE or AGENT_COULD_NOT_START: a service the save needs did not answer just then; try again in a minute.", pipelineInputShape, async (input) => json(await client.api("/api/loop/pipelines", { method: "POST", body: input })));
     server.tool("loop_update_pipeline", "Edit a pipeline: its name, use case, model, challenger, tags, when to train, promotion policy, version limit, or switch it off and on. Send ONLY what changes: an absent key leaves it alone, and null clears max_versions (no limit), challenger_model (no challenger) or train_when.schedule (train as soon as the minimum is met). Saving is the authorization, as on a create: a change that makes attempts cost more (a bigger model, switching it back on) is SPEND CONSENT, so say so first. Pass expected_revision (the pipeline's revision as you read it) to be refused REVISION_MISMATCH rather than overwrite someone else's edit. A model change while an attempt is running is refused RUN_ACTIVE. 409 NOT_A_SIMPLE_PIPELINE: this pipeline was created before pipelines had tags, a use case or a challenger, so those three cannot be edited; everything else can. Every other refusal is loop_create_pipeline's, with the same meaning.", {
         pipeline_id: z.string().describe("The pipeline's id (rule_id), from loop_list_pipelines"),
         ...pipelineInputShape,
@@ -3088,8 +3142,8 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         method: "PUT",
         body,
     })));
-    server.tool("loop_delete_pipeline", "Delete a pipeline for good. An attempt still running is cancelled; versions it made, their checkpoints and whatever serves the app now are untouched, and its samples and tags are kept. Recording from its own tag's source stops: an app still sending there gets captured:false. THIS CANNOT BE UNDONE and the name is spent: the response's retained_name can never be used by another pipeline in this workspace. To stop it for now, loop_update_pipeline with enabled false. Confirm with the user first.", { pipeline_id: z.string() }, async ({ pipeline_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(pipeline_id)}`, { method: "DELETE" })));
-    server.tool("loop_run_pipeline", "Train now: start an attempt without waiting for the schedule. SPEND CONSENT: it trains on real GPUs charged to the wallet, so say so and get a yes. It never skips the minimum: 422 NOT_ENOUGH_SAMPLES carries have and need — read both back. 409 RUN_ACTIVE: an attempt is already running. 409 VERSION_LIMIT_REACHED: raise or remove max_versions. 409 RULE_PAUSED: it is switched off or paused (paused_reason says why). 409 PIPELINE_NEEDS_SAVE: its settings changed without a pipeline save; save it with loop_update_pipeline, then train again. 409 MONTHLY_LIMIT_REACHED: the platform's runaway guard for this month; read resumes_at back. 409 SCORING_CAPPED: the comparison's key is at its monthly cap, so what it trained could not be scored; read resumes_at back. 503 AGENT_COULD_NOT_START or 409 AGENT_OFF: the key every attempt is scored with could not be renewed just then, so nothing was started; train now again in a minute (saving the pipeline does not help). Do not retry any of the others unchanged.", { pipeline_id: z.string() }, async ({ pipeline_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(pipeline_id)}/run`, {
+    server.tool("loop_delete_pipeline", "Delete a pipeline for good. An attempt still running is cancelled; the deployment its live version answers from is stopped and deleted, so the live version stops answering and is no longer billed, unless another pipeline or model name of yours answers from it too. If one does, that deployment keeps running and billed until you stop it on Deployments, and its trained checkpoint is kept (and billed as storage) until that deployment is deleted. Its history is deleted in the background after the call returns: the checkpoints its attempts trained, the sets they trained on, and every conversation of their comparison and benchmark reports. Its samples and tags are kept. Recording from its own tag's source stops: an app still sending there gets captured:false. THIS CANNOT BE UNDONE and the name is spent: the response's retained_name can never be used by another pipeline in this workspace. To stop it for now, loop_update_pipeline with enabled false. Confirm with the user first, and say that the live version stops answering and the history is deleted.", { pipeline_id: z.string() }, async ({ pipeline_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(pipeline_id)}`, { method: "DELETE" })));
+    server.tool("loop_run_pipeline", "Train now: start an attempt without waiting for the schedule. SPEND CONSENT: it trains on real GPUs charged to the wallet, so say so and get a yes. It never skips the minimum: 422 NOT_ENOUGH_SAMPLES carries have and need — read both back. 409 RUN_ACTIVE: an attempt is already running. An attempt waiting for GPUs to compare (active_run.state waiting_gpus) is the one exception: Train now books its two comparison machines again at once and answers 202; nothing is trained again. 409 VERSION_LIMIT_REACHED: raise or remove max_versions. 409 RULE_PAUSED: it is switched off or paused (paused_reason says why). 409 PIPELINE_NEEDS_SAVE: its settings changed without a pipeline save; save it with loop_update_pipeline, then train again. 409 MONTHLY_LIMIT_REACHED: the platform's runaway guard for this month; read resumes_at back. 409 SCORING_CAPPED: the comparison's key is at its monthly cap, so what it trained could not be scored; read resumes_at back (the pipeline's scoring_capped_until says the same before you try). 409 LIVE_VERSION_STOPPED: the deployment the live version answers from is stopped, paused or deleted, so an attempt could not be compared against it; the user resumes it in Deployments or rolls the version back. 503 AGENT_COULD_NOT_START or 409 AGENT_OFF: the key every attempt is scored with could not be renewed just then, so nothing was started; train now again in a minute (saving the pipeline does not help). Do not retry any of the others unchanged.", { pipeline_id: z.string() }, async ({ pipeline_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(pipeline_id)}/run`, {
         method: "POST",
         body: {},
     })));
@@ -3108,7 +3162,7 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         method: "POST",
         body,
     })));
-    server.tool("loop_list_training_runs", "Every attempt of every pipeline, newest first: what state it is in, what it has spent, and what the comparison said. A run sitting in 'awaiting_review' is waiting for a PERSON — it holds a candidate that is being paid for by the minute, so surface those first when the user asks what needs attention.", {
+    server.tool("loop_list_training_runs", "Every attempt of every pipeline, newest first: what state it is in, what it has spent, and what the comparison said. A run sitting in 'awaiting_review' is waiting for a PERSON — it holds a candidate that is being paid for by the minute, so surface those first when the user asks what needs attention. A run in 'waiting_gpus' trained and is waiting for GPUs to run both models side by side: each try can bill the machine that got a GPU for up to 20 minutes while it waits for the other, nothing bills between tries, it keeps what it trained, tries again after 30 minutes, then 1, 2 and 4 hours, and ends after four tries or 24 hours (its last_reason says when; a last_reason opening 'waiting for room in this workspace to compare' means the workspace is at its limit of running deployments, and stopping a deployment lets it run), and Train now tries again at once. billed_candidate_up_to true means billed_candidate_cents is 'up to': a comparison machine was charged in it, counted from its first boot while the platform bills from its first answer (and at the hourly cap when no price was reported), so say 'up to', never 'spent'.", {
         rule_id: z.string().optional().describe("Only this pipeline's attempts (its id, from loop_list_pipelines)"),
         state: z.string().optional().describe("Only runs in this state, e.g. 'awaiting_review'"),
         limit: z.number().int().min(1).max(200).optional(),
@@ -3116,11 +3170,11 @@ create (1-5 choices), and the queue itself stays opt-in.`,
     }, async ({ rule_id, state, limit, offset }) => json(await client.api("/api/loop/training-runs", {
         params: { rule_id, state, limit: limit?.toString(), offset: offset?.toString() },
     })));
-    server.tool("loop_get_training_run", "One run in full, with the timeline of every state it passed through and why, links to the job, the candidate, the datasets and the comparison, and available_actions — which says what THIS reader may actually do next. Use available_actions rather than guessing from the state: a run can look promotable and not be, because the consent moved or the candidate is already gone.", { run_id: z.string() }, async ({ run_id }) => json(await client.api(`/api/loop/training-runs/${encodeURIComponent(run_id)}`)));
-    server.tool("loop_list_pipelines", "Every pipeline in the workspace: the series of versions it has produced (versions are attempts that were put live; attempts[] lists every attempt, winner or not), which one serves the app today (live_version, the same as champion_version, null when what serves is not one of its versions), how each version did against the champion of its day and on the standing benchmark, versions_made against max_versions, the run in flight and which version it will become, and status. Read-only. When the user asks 'is this getting better' or 'what is it doing', answer from this, and read status back honestly: needs_review and needs_funds are waiting on THEM (a decision, a top-up), paused names why in paused_reason or needs_consent, complete means the version limit is reached, and running is the platform working. next_reason is the pipeline's own sentence about what it is waiting for. month_spent_cents against monthly_ceiling_cents is what decides whether the next run may start; month_spent_cents is what runs started this UTC month have cost at most (a run still in progress counts at its ceilings), so say 'up to', never 'spent'. A pipeline trains only on new samples: it never retrains the conversations an attempt already had with a recipe changed. The list is a page of 100, newest first: total is how many pipelines the workspace has and truncated is true when more exist after this page. When truncated, say it is the newest of total, and pass offset (this page's offset plus its length) for the next page.", { offset: z.number().int().min(0).optional().describe("How many of the newest pipelines to skip") }, async ({ offset }) => json(await client.api("/api/loop/pipelines", {
+    server.tool("loop_get_training_run", "One run in full: the base model it trained (base_model_id, model_family) and the recipe, the rows it trained on and held back, its GPU time and what it was billed (billed_candidate_up_to true means billed_candidate_cents is 'up to': a comparison machine was charged in it, counted from its first boot while the platform bills from its first answer, and at the hourly cap when no price was reported -- say 'up to', never 'spent'), the timeline of every state it passed through and why, links to the job, the candidate and the comparison, benchmark_runs (every benchmark it was replayed on; each run_id is the replay's id for loop_get_benchmark_run and loop_list_benchmark_run_items), and available_actions — which says what THIS reader may actually do next. Use available_actions rather than guessing from the state: a run can look promotable and not be, because the consent moved or the candidate is already gone.", { run_id: z.string() }, async ({ run_id }) => json(await client.api(`/api/loop/training-runs/${encodeURIComponent(run_id)}`)));
+    server.tool("loop_list_pipelines", "Every pipeline in the workspace: the series of versions it has produced (versions are attempts that were put live; attempts[] lists every attempt, winner or not), which one serves the app today (live_version, the same as champion_version, null when what serves is not one of its versions) and whether its deployment still answers (live_deployment.status: serving answers; starting is on its way up; stopped, paused_for_funds or deleted answers nothing and is not billed; paused_by_platform was paused because billing could not be reached and answers nothing; failed answers nothing; live_deployment null means no check is recorded yet or what serves is not a deployment -- never say a live version with any status but serving is answering the app), how each version did against the champion of its day and on the standing benchmark, versions_made against max_versions, the run in flight and which version it will become, and status. Read-only. When the user asks 'is this getting better' or 'what is it doing', answer from this, and read status back honestly: needs_review and needs_funds are waiting on THEM (a decision, a top-up), paused names why in paused_reason (consent_invalid means its settings changed since it was last saved: saving it again starts it), complete means the version limit is reached, and running is the platform working -- including an attempt whose active_run.state is waiting_gpus: it trained, but no GPUs were free to run the new model and the one it is compared with side by side, so both machines were stopped; each try can bill the machine that got a GPU for up to 20 minutes while it waits for the other, nothing bills between tries, it keeps what it trained, tries again after 30 minutes, then 1, 2 and 4 hours, and ends after four tries or 24 hours (its last_reason says when; a last_reason opening 'waiting for room in this workspace to compare' means the workspace is at its limit of running deployments, and stopping a deployment lets it run), and Train now tries again at once. cost_up_to true on an attempt, active_run or a version means its cost_cents is 'up to', the most it can have cost: a comparison machine was charged in it, counted from its first boot while the platform bills it from its first answer (and at the hourly cap when no price was reported), so say 'up to', never 'spent'. next_reason is the pipeline's own sentence about what it is waiting for. scoring_capped_until, when set, is when the workspace's scoring key comes off its monthly cap: until then nothing trains and Train now answers SCORING_CAPPED, so say so instead of offering Train now. month_spent_cents against monthly_ceiling_cents is what decides whether the next run may start; month_spent_cents is what runs started this UTC month have cost at most (a run still in progress counts at its ceilings), so say 'up to', never 'spent'. A pipeline trains only on new samples: it never retrains the conversations an attempt already had with a recipe changed. The list is a page of 100, newest first: total is how many pipelines the workspace has and truncated is true when more exist after this page. When truncated, say it is the newest of total, and pass offset (this page's offset plus its length) for the next page.", { offset: z.number().int().min(0).optional().describe("How many of the newest pipelines to skip") }, async ({ offset }) => json(await client.api("/api/loop/pipelines", {
         params: { offset: offset ? offset.toString() : undefined },
     })));
-    server.tool("loop_get_pipeline", "One pipeline in full, by its id: every version with its verdict, the decision a person made, its win rate and benchmark score, what it cost and the recipe it trained on (an old version whose trigger was 'variant' retrained the same conversations with one setting changed, named in recipe_note; variants are retired), plus the run in flight, and what this month's runs have cost at most (month_spent_cents, an upper bound, not an exact spend) against the monthly limit. Read-only: promote, reject and roll back stay on the run tools, and its settings on loop_update_pipeline. minimums says how many samples it has against what the next attempt needs. Only compare benchmark scores whose benchmark_comparable is true; the others were measured on a different yardstick. versions[] are WINNERS ONLY -- attempts that were put live, numbered v1, v2 ... in the order they went live, each with its attempt, base_model_id and model_family -- and live_version is the one serving now; attempts[] is every training run the pipeline made, winner or not, with its outcome (became_version, not_better, inconclusive, stopped, failed, running, waiting_for_review) and trigger (data, schedule, manual, or challenger for the second model trained on the same data). Nothing in flight is a version yet.", { rule_id: z.string().describe("The pipeline's id, from loop_list_pipelines") }, async ({ rule_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(rule_id)}`)));
+    server.tool("loop_get_pipeline", "One pipeline in full, by its id: every version with its verdict, the decision a person made, its win rate, its pass rates on the held-back samples (candidate_pass_rate and incumbent_pass_rate: the share each side passed, score 0.70 or more) and benchmark score, what it cost and why it was made (trigger, in the same words as attempts[]), plus the run in flight, and what this month's runs have cost at most (month_spent_cents, an upper bound, not an exact spend) against the monthly limit. Read-only: promote, reject and roll back stay on the run tools, and its settings on loop_update_pipeline. minimums says how many samples it has against what the next attempt needs. scoring_capped_until, when set, is when the workspace's scoring key comes off its monthly cap: until then nothing trains and Train now answers SCORING_CAPPED with that time as resumes_at. Only compare benchmark scores whose benchmark_comparable is true; the others were measured on a different yardstick. versions[] are WINNERS ONLY -- attempts that were put live, numbered v1, v2 ... in the order they went live, each with its attempt, base_model_id and model_family -- and live_version is the one the app's serving name reaches now, with live_deployment.status saying whether its deployment answers (serving answers; starting is on its way up; stopped, paused_for_funds or deleted answers nothing and is not billed; paused_by_platform was paused because billing could not be reached and answers nothing; failed answers nothing; null when no check is recorded yet or what serves is not a deployment); attempts[] is every training run the pipeline made, winner or not, with its outcome (became_version, not_better, inconclusive, stopped, failed, running, waiting_for_review) and trigger (data, schedule, manual, or challenger for the second model trained on the same data). Nothing in flight is a version yet. cost_up_to true on attempts[], active_run or versions[] means that cost_cents is 'up to', the most it can have cost: a comparison machine was charged in it, counted from its first boot while the platform bills from its first answer (and at the hourly cap when no price was reported), so say 'up to', never 'spent'. An active_run.state of waiting_gpus means the attempt trained and is waiting for GPUs to run the new model and the one it is compared with side by side: each try can bill the machine that got a GPU for up to 20 minutes while it waits for the other, nothing bills between tries, it keeps what it trained, tries again after 30 minutes, then 1, 2 and 4 hours, and ends after four tries or 24 hours (its last_reason says when; a last_reason opening 'waiting for room in this workspace to compare' means the workspace is at its limit of running deployments, and stopping a deployment lets it run), and Train now tries again at once.", { rule_id: z.string().describe("The pipeline's id, from loop_list_pipelines") }, async ({ rule_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(rule_id)}`)));
     server.tool("loop_list_pipeline_benchmarks", "The benchmarks a pipeline is measured on, in priority order (1 is the primary), then the ones it stopped using, and its promotion policy. Every attempt is replayed on each attached benchmark; the policy decides which of those measurements make it a version.", { pipeline_id: z.string() }, async ({ pipeline_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(pipeline_id)}/benchmarks`)));
     server.tool("loop_attach_pipeline_benchmark", "Measure a pipeline on one more benchmark (from loop_list_benchmarks or loop_create_benchmark), last unless priority is given; priority 1 makes it the primary. Applies from the next attempt; older versions are not re-measured and show not_measured for it. 409 BENCHMARK_ALREADY_ATTACHED or BENCHMARK_RETIRED.", {
         pipeline_id: z.string(),
@@ -3144,7 +3198,7 @@ create (1-5 choices), and the queue itself stays opt-in.`,
     }, async ({ pipeline_id, benchmark_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(pipeline_id)}/benchmarks/${encodeURIComponent(benchmark_id)}`, {
         method: "DELETE",
     })));
-    server.tool("loop_compare_versions", "A pipeline's versions side by side: each version's head-to-head result and its score on every benchmark, with comparable flags. Pass a and b (VERSION numbers, e.g. 1 and 5) for those two and a difference per benchmark; a benchmark attached after a version is not_measured for it, never a zero. Compare only scores marked comparable.", {
+    server.tool("loop_compare_versions", "A pipeline's versions side by side: each version's head-to-head result and its score on every benchmark, each with the share of conversations each side passed (incumbent_pass_rate, candidate_pass_rate; score 0.70 or more), with comparable flags. Pass a and b (VERSION numbers, e.g. 1 and 5) for those two and a difference per benchmark; a benchmark attached after a version is not_measured for it, never a zero. Compare only scores marked comparable.", {
         pipeline_id: z.string(),
         a: z.number().int().min(1).optional().describe("A version number"),
         b: z.number().int().min(1).optional().describe("Another version number"),
@@ -3166,7 +3220,7 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         method: "POST",
         body: { reason },
     })));
-    server.tool("loop_rollback_training_run", "Undo a promotion: point the serving name back at what answered before it. Available for 30 days, after which the answer is ROLLBACK_WINDOW_CLOSED and the way back is a new run. Tell the user what the name will be serving once this lands — the previous target is named on the run.", {
+    server.tool("loop_rollback_training_run", "Undo a promotion: point the serving name back at what answered before it. Available for 30 days, after which the answer is ROLLBACK_WINDOW_CLOSED and the way back is a new run. 409 RUN_ACTIVE: another attempt is being compared with this version right now; roll back once that comparison finishes, or cancel the attempt first. Tell the user what the name will be serving once this lands — the previous target is named on the run.", {
         run_id: z.string(),
         reason: z.string().optional().describe("Why. It goes on the run's timeline."),
     }, async ({ run_id, reason }) => json(await client.api(`/api/loop/training-runs/${encodeURIComponent(run_id)}/rollback`, {
@@ -3180,8 +3234,8 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         method: "POST",
         body: { reason },
     })));
-    server.tool("loop_get_evaluation", "The comparison report behind a verdict: how many pairs were scored, the candidate's win rate, the mean scores per rubric dimension, the warnings, and the margins the verdict was measured against. Read the WARNINGS out too, not just the verdict — 'better' from a judge that rarely agrees with this workspace's reviewers, or from a handful of rows, is a number with a caveat attached, and the caveat is in that list.", { evaluation_id: z.string().describe("From the run's links, or loop_get_training_run") }, async ({ evaluation_id }) => json(await client.api(`/api/loop/evaluations/${encodeURIComponent(evaluation_id)}`)));
-    server.tool("loop_list_evaluation_items", "The paired conversations behind the numbers: the same prompt, what the version serving today answered, what the attempt answered, and what the judge made of each. This is how a person checks a verdict instead of trusting it. Filter to winner='serving' to read the pairs the attempt LOST (the items themselves say winner 'incumbent'), which is the honest thing to show somebody before they promote.", {
+    server.tool("loop_get_evaluation", "The comparison report behind a verdict: how many pairs were scored, the candidate's win rate, how many conversations each side passed (score 0.70 or more: candidate_passed, incumbent_passed, and the pass rates over rows_scored, all four null until the report is finished), the mean scores per rubric dimension, the warnings, and the margins the verdict was measured against. Read the WARNINGS out too, not just the verdict — 'better' from a judge that rarely agrees with this workspace's reviewers, or from a handful of rows, is a number with a caveat attached, and the caveat is in that list.", { evaluation_id: z.string().describe("From the run's links, or loop_get_training_run") }, async ({ evaluation_id }) => json(await client.api(`/api/loop/evaluations/${encodeURIComponent(evaluation_id)}`)));
+    server.tool("loop_list_evaluation_items", "The paired conversations behind the numbers: the same prompt, the reference a person gave for it (null when none), what the version serving today answered, what the attempt answered, what the judge made of each (judge.overall, judge.dimensions and judge.reason, its reason in its own words) and whether each passed (score 0.70 or more), and pairwise: the direct comparison asked in both orders, each with its reason, where a win counts only when both orders agree. This is how a person checks a verdict instead of trusting it. Filter to winner='serving' to read the pairs the attempt LOST (the items themselves say winner 'incumbent'), which is the honest thing to show somebody before they promote.", {
         evaluation_id: z.string(),
         winner: z.enum(["candidate", "serving", "tie"]).optional().describe("Only pairs with this outcome: candidate (the attempt won), serving (the version serving today won) or tie"),
         limit: z.number().int().min(1).max(200).optional().describe("50 when omitted, at most 200"),
@@ -3228,7 +3282,15 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         method: "POST",
         body: { reason },
     })));
-    server.tool("loop_get_benchmark_run", "One replay in full: both scores, the difference, the per-dimension breakdown and what it cost. Read status first — a replay that did not score every pinned conversation publishes no score.", { benchmark_run_id: z.string() }, async ({ benchmark_run_id }) => json(await client.api(`/api/loop/benchmark-runs/${encodeURIComponent(benchmark_run_id)}`)));
+    server.tool("loop_get_benchmark_run", "One replay in full: both scores, the difference, the per-dimension breakdown, how many conversations each side passed (score 0.70 or more) and the pass rates, the conversations the attempt won, lost and tied (wins, losses, ties), and what it cost. Read status first — a replay that did not score every pinned conversation publishes no score, and its pass counts, rates and wins are null too. loop_list_benchmark_run_items reads every conversation behind it.", { benchmark_run_id: z.string() }, async ({ benchmark_run_id }) => json(await client.api(`/api/loop/benchmark-runs/${encodeURIComponent(benchmark_run_id)}`)));
+    server.tool("loop_list_benchmark_run_items", "The conversations behind one replay, in the benchmark's order: the pinned prompt and its reference (null when none), what the version serving today answered, what the attempt answered, what the judge made of each (judge.overall, judge.dimensions and judge.reason) and whether each passed (score 0.70 or more), and which side won it (winner compares the two scores; a difference inside the tie band promotion uses is a tie, and it is null until both are scored). Filter to winner='serving' to read the conversations the attempt LOST (the items themselves say winner 'incumbent'). A replay that stopped before it began has no conversations: read its status_reason back instead.", {
+        benchmark_run_id: z.string().describe("The replay's id: from loop_get_benchmark_history, loop_compare_versions, or a run's benchmark_runs"),
+        winner: z.enum(["candidate", "serving", "tie"]).optional().describe("Only conversations with this outcome: candidate (the attempt won), serving (the version serving today won) or tie"),
+        limit: z.number().int().min(1).max(200).optional().describe("50 when omitted, at most 200"),
+        offset: z.number().int().min(0).optional(),
+    }, async ({ benchmark_run_id, winner, limit, offset }) => json(await client.api(`/api/loop/benchmark-runs/${encodeURIComponent(benchmark_run_id)}/items`, {
+        params: { winner, limit: limit?.toString(), offset: offset?.toString() },
+    })));
     server.tool("set_inference_alias", "Point a re-pointable public handle at a deployment. An alias is a NAME customers call that can be moved to different weights without them changing anything — it is what a promotion writes, and what lets a retrained model take over traffic with no client edit. Setting one CHANGES WHAT REAL TRAFFIC REACHES, so confirm with the user first. A name that belongs to a live deployment is refused (ALIAS_NAME_IS_A_DEPLOYMENT) unless the alias is being given that deployment's own name, and a target that is not serving is refused (TARGET_NOT_SERVABLE) rather than pointing the handle at something that cannot answer.", {
         name: z.string().describe("The public handle, e.g. 'support-bot'"),
         target_inference_id: z.string().describe("The deployment the name should reach, from list_inferences"),