runbios-mcp 0.2.20 → 0.2.21-dev.294

package/dist/server.js

@@ -607,6 +607,32 @@ export function summarizeInferenceStatus(data) {
     }
     return shaped;
 }
+const FINAL_MODEL_DELETED_BY = {
+    "system:pipeline_deleted": "when the Conscious Loop pipeline that ran it was deleted",
+    "system:save_total_limit": "by this run's checkpoint limit",
+    you: "by you",
+    member: "by a member of this workspace",
+};
+/**
+ * What a completed run's final model is now, from final_artifact_state in the
+ * same response. A model that was saved and verified and later deleted on
+ * purpose is "deleted", never a failed save. Null when the response does not
+ * say (a run that is not completed, or an older backend).
+ */
+function summarizeFinalModel(payload) {
+    const state = nonEmptyString(payload.final_artifact_state);
+    if (payload.status !== "completed" || !state)
+        return null;
+    const deletedAt = nonEmptyString(payload.final_artifact_deleted_at) ?? null;
+    const deletedBy = nonEmptyString(payload.final_artifact_deleted_by) ?? null;
+    const byWhom = deletedBy ? (FINAL_MODEL_DELETED_BY[deletedBy] ?? (deletedBy.startsWith("system:") ? "by the platform" : null)) : null;
+    const note = state === "verified"
+        ? "The final model is saved and verified; it can be deployed or downloaded."
+        : state === "deleted"
+            ? `Training succeeded and its final model was saved and verified, then deleted${deletedAt ? ` at ${deletedAt}` : ""}${byWhom ? ` ${byWhom}` : ""}. It can no longer be deployed or downloaded; this is not a failed save.`
+            : "Training finished, but no committed final model could be verified. Do not deploy earlier checkpoints as this run's final model.";
+    return { state, deployable: state === "verified", deleted_at: deletedAt, deleted_by: deletedBy, note };
+}
 const TRAINING_PHASE_LABELS = {
     dataset_download: "downloading datasets",
     model_download: "downloading the model",
@@ -627,12 +653,14 @@ const TRAINING_PHASE_LABELS = {
  * value that is absent stays null, never guessed.
  */
 export function summarizeTrainingProgress(data) {
-    const payload = asRecord(data);
-    if (!payload)
+    const raw = asRecord(data);
+    if (!raw)
         return data;
+    const finalModel = summarizeFinalModel(raw);
+    const payload = finalModel ? { ...raw, final_model: finalModel } : raw;
     const phases = Array.isArray(payload.phases) ? payload.phases.map(asRecord).filter(Boolean) : [];
     if (phases.length === 0)
-        return data;
+        return finalModel ? payload : data;
     const active = phases.find((p) => p.state === "active") ?? null;
     const num = (v) => (typeof v === "number" && Number.isFinite(v) ? v : null);
     const phaseId = active ? nonEmptyString(active.phase) : undefined;
@@ -805,18 +833,18 @@ pipeline is deleted. Every other source is switched on with loop_set_capture.
 4. loop_add_signal / loop_list_signals - Feedback: accepted (good), rejected (bad), or edited with the corrected answer. Append-only. A correction is the most valuable feedback there is.
 5. loop_label_trace / loop_remove_label / loop_list_labels / loop_get_trace_pipelines - Tags. A pipeline trains on every conversation carrying any of its tags, and a conversation can carry several and feed several pipelines; loop_get_trace_pipelines says which pipelines one conversation feeds.
 6. loop_stats / loop_get_metrics_history - How much is recorded and reviewed, and how the attempts and the feedback moved over time.
-7. loop_list_models - The base models a pipeline can train, and the sample floors. The only list a pipeline's model may come from.
-8. loop_create_pipeline - One use case: a name, a model (optionally a challenger model), its tags, when to train (at least N new samples, and optionally a schedule) and how a new version is promoted. It switches on the source named by its own tag, so an app can send samples at once. SPEND CONSENT: saving it authorises attempts on the workspace's wallet whenever its data has grown enough; the platform chooses the GPUs and only the GPU time used is charged. Say so and wait for a yes.
-9. loop_list_pipelines / loop_get_pipeline / loop_update_pipeline / loop_delete_pipeline - Each pipeline's versions (winners only, v1, v2 ... numbers only go up), every attempt and how it ended, the live version, status and next_reason, and its settings.
+7. loop_list_models - The base models a pipeline can train, and the platform's fixed sample minimums (floors). The only list a pipeline's model may come from.
+8. loop_create_pipeline - One use case: a name, a model (optionally a challenger model), its tags, when to train (optionally a schedule; the sample minimums are fixed by the platform) and how a new version is promoted. It switches on the source named by its own tag, so an app can send samples at once. SPEND CONSENT: saving it authorises attempts on the workspace's wallet whenever its data has grown enough; the platform chooses the GPUs and only the GPU time used is charged. Say so and wait for a yes.
+9. loop_list_pipelines / loop_get_pipeline / loop_update_pipeline / loop_delete_pipeline - Each pipeline's versions (winners only, v1, v2 ... numbers only go up), every attempt and how it ended, the live version and whether its deployment answers (live_deployment), status and next_reason, and its settings. Deleting a pipeline stops its live version's deployment (unless another pipeline or model name answers from it too) and deletes its history.
 10. loop_run_pipeline / loop_import_into_pipeline - Train now (skips the schedule, never the minimum), and bring a file in as this pipeline's samples.
-11. loop_create_benchmark / loop_list_benchmarks / loop_get_benchmark / loop_list_benchmark_items / loop_get_benchmark_history / loop_retire_benchmark / loop_get_benchmark_run - Fixed sets of conversations every attempt is replayed on, so "is it getting better" has a yardstick that does not move.
-12. loop_list_pipeline_benchmarks / loop_attach_pipeline_benchmark / loop_move_pipeline_benchmark / loop_remove_pipeline_benchmark / loop_compare_versions - A pipeline's benchmarks in priority order (1 is the primary), and its versions side by side (v1 against v5).
-13. loop_list_training_runs / loop_get_training_run - Every attempt: its timeline, and available_actions — what this reader may actually do next, rather than what the state looks like it allows.
-14. loop_get_evaluation / loop_list_evaluation_items - The comparison behind a verdict: the win rate and the per-dimension means on the held-back samples, and the paired conversations themselves. Read the warnings out, not just the verdict.
+11. loop_create_benchmark / loop_list_benchmarks / loop_get_benchmark / loop_list_benchmark_items / loop_get_benchmark_history / loop_retire_benchmark / loop_get_benchmark_run / loop_list_benchmark_run_items - Fixed sets of conversations every attempt is replayed on, so "is it getting better" has a yardstick that does not move, and each replay's report: its scores, how many conversations each model passed (score 0.70 or more), and every conversation behind them.
+12. loop_list_pipeline_benchmarks / loop_attach_pipeline_benchmark / loop_move_pipeline_benchmark / loop_remove_pipeline_benchmark / loop_compare_versions / loop_compare_attempts - A pipeline's benchmarks in priority order (1 is the primary), its versions side by side (v1 against v5), and any two attempts side by side, put live or not, conversation by conversation on the held-back samples both were scored on.
+13. loop_list_training_runs / loop_get_training_run - Every attempt: its timeline, why it went live or did not (promotion, and verdict_checks: every bar its comparison needed, what it got and whether it was met), what each part cost, and available_actions — what this reader may actually do next, rather than what the state looks like it allows.
+14. loop_get_evaluation / loop_list_evaluation_items - The comparison behind a verdict: the win rate, the pass rates (the share of conversations each model passed: score 0.70 or more) and the per-dimension means on the held-back samples, and the paired conversations themselves, with the judge's reasons. Read the warnings out, not just the verdict.
 15. loop_promote_training_run / loop_reject_training_run / loop_rollback_training_run / loop_cancel_training_run - The decision. Promotion makes an attempt the live version and changes what real customers get; it is undoable for 30 days.
 16. set_inference_alias / list_inference_aliases / delete_inference_alias - The re-pointable public handles a promotion writes: a name customers call that can be moved to different weights without them changing anything.
 
-END TO END: capture what a model was asked and answered (or import it), say whether each answer was good, bad or corrected, and tag it; create ONE pipeline per use case. When at least min_samples new samples carry its tags (and its schedule is due, if it has one) the platform trains an attempt, compares it with the live version on held-back samples it never trains on and on the pipeline's benchmarks, and promotes it only when it is better by the promotion policy — so each version beats the one before it. Everything before the pipeline costs nothing; saving a pipeline is where money starts, which is why it is spend consent. Two fields carry the state a user actually asks about and BOTH must be read back in plain words rather than summarised as 'fine': status says whether the platform is working or the user has to act, and next_reason says what a quiet pipeline is waiting for, because a pipeline waiting for samples looks identical to a broken one.`;
+END TO END: capture what a model was asked and answered (or import it), say whether each answer was good, bad or corrected, and tag it; create ONE pipeline per use case. When enough new samples carry its tags to meet the platform's fixed minimum (loop_list_models floors; the pipeline's minimums say how many are still needed) and its schedule is due, if it has one, the platform trains an attempt, compares it with the live version on held-back samples it never trains on and on the pipeline's benchmarks, and promotes it only when it is better by the promotion policy — so each version beats the one before it. Everything before the pipeline costs nothing; saving a pipeline is where money starts, which is why it is spend consent. Two fields carry the state a user actually asks about and BOTH must be read back in plain words rather than summarised as 'fine': status says whether the platform is working or the user has to act, and next_reason says what a quiet pipeline is waiting for, because a pipeline waiting for samples looks identical to a broken one.`;
     const server = {
         tool(name, description, paramsSchema, cb) {
             if (allowedTools && !allowedTools.has(name))
@@ -3092,8 +3120,9 @@ create (1-5 choices), and the queue itself stays opt-in.`,
     /*                                                                           */
     /*  A PIPELINE is one use case: a name, a model, the tags whose samples it    */
     /*  trains on, when to train (at least N new samples, and optionally a        */
-    /*  schedule) and how a new version is promoted. The platform decides the     */
-    /*  rest -- the GPUs, LoRA, the held-back set, the comparison -- and SAVING A */
+    /*  schedule), the share of its samples held back to judge each version, and  */
+    /*  how a new version is promoted. The platform decides the rest -- the GPUs, */
+    /*  LoRA, the held-back floor, the comparison -- and SAVING A                 */
     /*  PIPELINE IS THE AUTHORIZATION TO SPEND: whenever its data has grown       */
     /*  enough it trains an attempt on the workspace's wallet with nobody in the  */
     /*  room. Only the GPU time an attempt actually uses is charged.              */
@@ -3104,7 +3133,7 @@ create (1-5 choices), and the queue itself stays opt-in.`,
     /*  train_when.schedule), so omit what you are not changing.                  */
     /* ══════════════════════════════════════════════════════════════════════════ */
     const pipelineModelSchema = z.object({
-        id: z.string().describe("A model id from loop_list_models. Any other model is refused MODEL_NOT_TRAINABLE."),
+        id: z.string().describe("A model id from loop_list_models. On a create, any other model is refused MODEL_NOT_TRAINABLE. On an edit only a changed id is checked against the list: an id equal to the one the pipeline has there now (ignoring case) is kept, and whether it can still be trained is checked when an attempt starts."),
     });
     /** The pipeline create and edit body: every field the service accepts, and no other. */
     const pipelineInputShape = {
@@ -3112,26 +3141,35 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         name: z.string().optional().describe("What the pipeline is called, e.g. 'Support replies'. Its own tag is made from it and does not change afterwards."),
         use_case: z.string().optional().describe("One sentence on what it is for"),
         model: pipelineModelSchema.optional().describe("The base model to improve"),
-        challenger_model: pipelineModelSchema.nullable().optional().describe("Optional second model: after each attempt it is trained on the same data, and it too must beat the live version to become one. null removes it."),
+        challenger_model: pipelineModelSchema.nullable().optional().describe("Optional second model: after each attempt it is trained on the same data, and it too must beat the live version to become one. When it goes live it becomes the pipeline's model and the current model becomes the second model. null removes it; an attempt of it already running is not stopped."),
         tags: z.array(z.string()).optional().describe("More tags whose samples this pipeline trains on, beside its own. On an edit the list replaces the previous one; the own tag stays."),
+        // STRICT: the sample minimums are fixed by the platform, and the service
+        // refuses a train_when.min_samples by name. zod would otherwise strip the
+        // key in silence, the save would succeed, and the agent would tell the user
+        // it had set a minimum nothing set.
         train_when: z.object({
-            min_samples: z.number().int().min(1).optional().describe("Train when at least this many NEW samples exist. The default and the floor are loop_list_models' floors.new_samples; below it is refused MIN_SAMPLES_TOO_LOW. The first version also needs floors.first_version_samples."),
             schedule: z.object({
                 every: z.enum(["day", "week"]),
                 weekday: z.number().int().min(0).max(6).optional().describe("0 is Sunday. Required for 'week'; leave it out for 'day'."),
                 hour_utc: z.number().int().min(0).max(23).describe("The UTC hour"),
-            }).nullable().optional().describe("Also wait for this time. Training starts only when BOTH the minimum and the schedule are met. null trains as soon as the minimum is met."),
-        }).optional(),
+            }).nullable().optional().describe("Also wait for this time. Training starts only when BOTH the platform's fixed sample minimum and the schedule are met. null trains as soon as the minimum is met."),
+        }).strict().optional().describe("When to train, beyond the platform's fixed sample minimum: only a schedule. There is no min_samples: the minimums are fixed (loop_list_models floors) and cannot be set."),
         promotion: z.object({
             policy: z.enum(["holdout", "all", "primary", "k_of_n", "weighted", "manual"]).describe("holdout (default): better on the held-back samples. all / primary / k_of_n / weighted: by the pipeline's benchmarks. manual: never promoted without a person."),
             k: z.number().int().min(1).optional().describe("For k_of_n only: how many measurements must improve"),
         }).optional(),
+        // STRICT, and never null: the share is the member's, the floor of 100 and
+        // the count are the platform's, and the service refuses anything else by
+        // name. zod would otherwise strip holdout.count in silence.
+        holdout: z.object({
+            percent: z.number().int().min(5).max(20).describe("The share of the pipeline's samples held back to judge each attempt and never trained on: a whole number from 5 to 20 (5 is the default). At least 100 are held back whatever it says."),
+        }).strict().optional().describe("The held-back share (loop_list_models floors.held_back says the range). It does not change what the pipeline may spend and nothing is asked again: a new share is a new revision, saved at once. But a larger share means more held-back samples are scored on each attempt (at most floors.held_back.scored_rows, the same ones every time), and scoring is paid from the wallet; a share that holds back more than that only keeps samples out of training. Lowering it keeps every sample already held back; raising it holds back only new samples, and new samples fill the held-back set before they count toward the next attempt, so after a raise the next attempt waits for them (the pipeline's minimums say how many). Leave it out to keep the pipeline's (or the default on a create)."),
         max_versions: z.number().int().min(1).max(100).nullable().optional().describe("How many versions it may make, 1 to 100; null (the default) is no limit. A version is an attempt that became the live model; attempts that lost are not counted."),
         enabled: z.boolean().optional().describe("false saves it switched off"),
     };
-    server.tool("loop_list_models", "The base models a pipeline can train — the ONLY models loop_create_pipeline accepts — recommended first, each with its name, author, family and size, and floors: first_version_samples (what a new pipeline's first attempt needs) and new_samples (the default and smallest train_when.min_samples). Offer the user a choice from this list; never type a model id from memory.", {}, async () => json(await client.api("/api/loop/models")));
-    server.tool("loop_create_pipeline", "Create a pipeline: one use case, trained again whenever enough new samples carrying its tags exist. SPEND CONSENT: saving it authorises the platform to train attempts on the workspace's wallet with nobody present — whenever train_when is met it trains on GPUs the platform chooses, compares the attempt with the live version, and promotes it by the promotion policy. Only the GPU time actually used is charged. Say that in plain words and WAIT FOR AN EXPLICIT YES before calling. Take model.id from loop_list_models, never from memory. Nothing is saved when it is refused. 409 PIPELINE_NAME_TAKEN or OWN_TAG_TAKEN: choose another name. 422 MIN_SAMPLES_TOO_LOW, NO_GPU_FITS, MODEL_NOT_TRAINABLE or PROMOTION_*: the message names the fix. 422 NO_JUDGE_MODEL: no listed model can score the attempts; tell the user rather than retrying. 422 MODEL_REVISION_UNAVAILABLE: the model could not be pinned because the training service did not answer; try again shortly. 422 METHOD_NOT_ENABLED or CEILING_TOO_LOW: the platform cannot train this model now and nothing in the request changes that; tell the user. 409 TRAINING_NOT_ENABLED: training is in beta and the organisation is not enabled yet — tell the user the platform team enables it, and do not retry. 503 MODELS_UNAVAILABLE, GPU_OPTIONS_UNAVAILABLE or AGENT_COULD_NOT_START: a service the save needs did not answer just then; try again in a minute.", pipelineInputShape, async (input) => json(await client.api("/api/loop/pipelines", { method: "POST", body: input })));
-    server.tool("loop_update_pipeline", "Edit a pipeline: its name, use case, model, challenger, tags, when to train, promotion policy, version limit, or switch it off and on. Send ONLY what changes: an absent key leaves it alone, and null clears max_versions (no limit), challenger_model (no challenger) or train_when.schedule (train as soon as the minimum is met). Saving is the authorization, as on a create: a change that makes attempts cost more (a bigger model, switching it back on) is SPEND CONSENT, so say so first. Pass expected_revision (the pipeline's revision as you read it) to be refused REVISION_MISMATCH rather than overwrite someone else's edit. A model change while an attempt is running is refused RUN_ACTIVE. 409 NOT_A_SIMPLE_PIPELINE: this pipeline was created before pipelines had tags, a use case or a challenger, so those three cannot be edited; everything else can. Every other refusal is loop_create_pipeline's, with the same meaning.", {
+    server.tool("loop_list_models", "The base models a pipeline can train — the ONLY models loop_create_pipeline accepts — recommended first, each with its name, author, family and size, and floors, the platform's fixed sample minimums (read-only; no pipeline sets them): first_version_samples (what a new pipeline's first attempt needs at the default held-back share, the held-back samples included), new_samples and new_samples_percent (each later attempt waits for new_samples new samples, or new_samples_percent percent of the samples the last trained attempt trained on when that is more), and held_back (at least min_rows samples are held back, or the share a pipeline chooses from default_percent to max_percent if that is more; each attempt is judged on at most scored_rows of them, so a share that holds back more only keeps samples out of training). Offer the user a choice from this list; never type a model id from memory. 503 MODELS_UNAVAILABLE: the list could not be read just then; try again in a minute.", {}, async () => json(await client.api("/api/loop/models")));
+    server.tool("loop_create_pipeline", "Create a pipeline: one use case, trained again whenever enough new samples carrying its tags exist. SPEND CONSENT: saving it authorises the platform to train attempts on the workspace's wallet with nobody present — whenever train_when is met it trains on GPUs the platform chooses, compares the attempt with the live version, and promotes it by the promotion policy. Only the GPU time actually used is charged. Say that in plain words and WAIT FOR AN EXPLICIT YES before calling. Take model.id from loop_list_models, never from memory. Nothing is saved when it is refused. 409 PIPELINE_NAME_TAKEN or OWN_TAG_TAKEN: choose another name. 422 NO_GPU_FITS, MODEL_NOT_TRAINABLE or PROMOTION_*: the message names the fix. 422 NO_JUDGE_MODEL: no listed model can score the attempts; tell the user rather than retrying. 422 MODEL_REVISION_UNAVAILABLE: the model could not be pinned because the training service did not answer; try again shortly. 422 METHOD_NOT_ENABLED or CEILING_TOO_LOW: the platform cannot train this model now and nothing in the request changes that; tell the user. 409 TRAINING_NOT_ENABLED: training is in beta and the organisation is not enabled yet — tell the user the platform team enables it, and do not retry. 503 MODELS_UNAVAILABLE, GPU_OPTIONS_UNAVAILABLE or AGENT_COULD_NOT_START: a service the save needs did not answer just then; try again in a minute.", pipelineInputShape, async (input) => json(await client.api("/api/loop/pipelines", { method: "POST", body: input })));
+    server.tool("loop_update_pipeline", "Edit a pipeline: its name, use case, model, challenger, tags, when to train, held-back share, promotion policy, version limit, or switch it off and on. Send ONLY what changes: an absent key leaves it alone, and null clears max_versions (no limit), challenger_model (no challenger) or train_when.schedule (train as soon as the minimum is met). Saving is the authorization, as on a create: a change that makes attempts cost more (a bigger model, switching it back on) is SPEND CONSENT, so say so first. A larger held-back share is not SPEND CONSENT (nothing is asked again and the limits do not move), and what it changes depends on how many are held back: while fewer than floors.held_back.scored_rows (1,500) are, each attempt can score more of them, paid from the wallet; once that many are, a larger share scores nothing more and costs nothing more, and only keeps more new samples out of training, for good; and a share that still comes to the floor of floors.held_back.min_rows (100) changes nothing. Say whichever applies, never that a raise always costs more. Pass expected_revision (the pipeline's revision as you read it) to be refused REVISION_MISMATCH rather than overwrite someone else's edit; REVISION_MISMATCH also comes back, sent or not, when the pipeline changed while the edit was being saved (a version went live, which can change its models): nothing was saved, so read it again and redo the edit. A model change while an attempt is running is refused RUN_ACTIVE. 409 NOT_A_SIMPLE_PIPELINE: this pipeline was created before pipelines had tags, a use case or a challenger, so those three cannot be edited; everything else can. Every other refusal is loop_create_pipeline's, with the same meaning.", {
         pipeline_id: z.string().describe("The pipeline's id (rule_id), from loop_list_pipelines"),
         ...pipelineInputShape,
         expected_revision: z.number().int().min(0).optional().describe("The revision you read"),
@@ -3139,7 +3177,7 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         method: "PUT",
         body,
     })));
-    server.tool("loop_delete_pipeline", "Delete a pipeline for good. An attempt still running is cancelled; versions it made, their checkpoints and whatever serves the app now are untouched, and its samples and tags are kept. Recording from its own tag's source stops: an app still sending there gets captured:false. THIS CANNOT BE UNDONE and the name is spent: the response's retained_name can never be used by another pipeline in this workspace. To stop it for now, loop_update_pipeline with enabled false. Confirm with the user first.", { pipeline_id: z.string() }, async ({ pipeline_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(pipeline_id)}`, { method: "DELETE" })));
+    server.tool("loop_delete_pipeline", "Delete a pipeline for good. An attempt still running is cancelled; the deployment its live version answers from is stopped and deleted, so the live version stops answering and is no longer billed, unless another pipeline or model name of yours answers from it too. If one does, that deployment keeps running and billed until you stop it on Deployments, and its trained checkpoint is kept (and billed as storage) until that deployment is deleted. Its history is deleted in the background after the call returns: the checkpoints its attempts trained, the sets they trained on, and every conversation of their comparison and benchmark reports. Its samples and tags are kept. Recording from its own tag's source stops: an app still sending there gets captured:false. THIS CANNOT BE UNDONE and the name is spent: the response's retained_name can never be used by another pipeline in this workspace. To stop it for now, loop_update_pipeline with enabled false. Confirm with the user first, and say that the live version stops answering and the history is deleted.", { pipeline_id: z.string() }, async ({ pipeline_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(pipeline_id)}`, { method: "DELETE" })));
     server.tool("loop_run_pipeline", "Train now: start an attempt without waiting for the schedule. SPEND CONSENT: it trains on real GPUs charged to the wallet, so say so and get a yes. It never skips the minimum: 422 NOT_ENOUGH_SAMPLES carries have and need — read both back. 409 RUN_ACTIVE: an attempt is already running. An attempt waiting for GPUs to compare (active_run.state waiting_gpus) is the one exception: Train now books its two comparison machines again at once and answers 202; nothing is trained again. 409 VERSION_LIMIT_REACHED: raise or remove max_versions. 409 RULE_PAUSED: it is switched off or paused (paused_reason says why). 409 PIPELINE_NEEDS_SAVE: its settings changed without a pipeline save; save it with loop_update_pipeline, then train again. 409 MONTHLY_LIMIT_REACHED: the platform's runaway guard for this month; read resumes_at back. 409 SCORING_CAPPED: the comparison's key is at its monthly cap, so what it trained could not be scored; read resumes_at back (the pipeline's scoring_capped_until says the same before you try). 409 LIVE_VERSION_STOPPED: the deployment the live version answers from is stopped, paused or deleted, so an attempt could not be compared against it; the user resumes it in Deployments or rolls the version back. 503 AGENT_COULD_NOT_START or 409 AGENT_OFF: the key every attempt is scored with could not be renewed just then, so nothing was started; train now again in a minute (saving the pipeline does not help). Do not retry any of the others unchanged.", { pipeline_id: z.string() }, async ({ pipeline_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(pipeline_id)}/run`, {
         method: "POST",
         body: {},
@@ -3167,12 +3205,12 @@ create (1-5 choices), and the queue itself stays opt-in.`,
     }, async ({ rule_id, state, limit, offset }) => json(await client.api("/api/loop/training-runs", {
         params: { rule_id, state, limit: limit?.toString(), offset: offset?.toString() },
     })));
-    server.tool("loop_get_training_run", "One run in full: the base model it trained (base_model_id, model_family) and the recipe, the rows it trained on and held back, its GPU time and what it was billed (billed_candidate_up_to true means billed_candidate_cents is 'up to': a comparison machine was charged in it, counted from its first boot while the platform bills from its first answer, and at the hourly cap when no price was reported -- say 'up to', never 'spent'), the timeline of every state it passed through and why, links to the job, the candidate and the comparison, and available_actions — which says what THIS reader may actually do next. Use available_actions rather than guessing from the state: a run can look promotable and not be, because the consent moved or the candidate is already gone.", { run_id: z.string() }, async ({ run_id }) => json(await client.api(`/api/loop/training-runs/${encodeURIComponent(run_id)}`)));
-    server.tool("loop_list_pipelines", "Every pipeline in the workspace: the series of versions it has produced (versions are attempts that were put live; attempts[] lists every attempt, winner or not), which one serves the app today (live_version, the same as champion_version, null when what serves is not one of its versions), how each version did against the champion of its day and on the standing benchmark, versions_made against max_versions, the run in flight and which version it will become, and status. Read-only. When the user asks 'is this getting better' or 'what is it doing', answer from this, and read status back honestly: needs_review and needs_funds are waiting on THEM (a decision, a top-up), paused names why in paused_reason (consent_invalid means its settings changed since it was last saved: saving it again starts it), complete means the version limit is reached, and running is the platform working -- including an attempt whose active_run.state is waiting_gpus: it trained, but no GPUs were free to run the new model and the one it is compared with side by side, so both machines were stopped; each try can bill the machine that got a GPU for up to 20 minutes while it waits for the other, nothing bills between tries, it keeps what it trained, tries again after 30 minutes, then 1, 2 and 4 hours, and ends after four tries or 24 hours (its last_reason says when; a last_reason opening 'waiting for room in this workspace to compare' means the workspace is at its limit of running deployments, and stopping a deployment lets it run), and Train now tries again at once. cost_up_to true on an attempt, active_run or a version means its cost_cents is 'up to', the most it can have cost: a comparison machine was charged in it, counted from its first boot while the platform bills it from its first answer (and at the hourly cap when no price was reported), so say 'up to', never 'spent'. next_reason is the pipeline's own sentence about what it is waiting for. scoring_capped_until, when set, is when the workspace's scoring key comes off its monthly cap: until then nothing trains and Train now answers SCORING_CAPPED, so say so instead of offering Train now. month_spent_cents against monthly_ceiling_cents is what decides whether the next run may start; month_spent_cents is what runs started this UTC month have cost at most (a run still in progress counts at its ceilings), so say 'up to', never 'spent'. A pipeline trains only on new samples: it never retrains the conversations an attempt already had with a recipe changed. The list is a page of 100, newest first: total is how many pipelines the workspace has and truncated is true when more exist after this page. When truncated, say it is the newest of total, and pass offset (this page's offset plus its length) for the next page.", { offset: z.number().int().min(0).optional().describe("How many of the newest pipelines to skip") }, async ({ offset }) => json(await client.api("/api/loop/pipelines", {
+    server.tool("loop_get_training_run", "One run in full: the base model it trained (base_model_id, model_family) and the recipe, the rows it trained on and held back, its GPU time and what it was billed (billed_candidate_up_to true means billed_candidate_cents is 'up to': a comparison machine was charged in it, counted from its first boot while the platform bills from its first answer, and at the hourly cap when no price was reported -- say 'up to', never 'spent'), the timeline of every state it passed through and why, links to the job, the candidate and the comparison, benchmark_runs (every benchmark it was replayed on; each run_id is the replay's id for loop_get_benchmark_run and loop_list_benchmark_run_items), promotion (outcome as the pipeline's attempts list says it; decision, where rejected means a member chose not to publish it even when the verdict says better, auto_rejected that the pipeline's rule turned it down on its own, and expired that nobody decided before the review deadline -- an expired attempt whose verdict is better was better, never say the rule refused it; and the promotion rule's reason and every measurement it read, each better, not_better, worse or not_measured with its reason -- null and empty when the rule recorded nothing), verdict_checks (every bar the held-back comparison had to clear -- conversations compared, failures, a rubric set, mean change, win rate, and the chance a model no better would win this often -- each with needed, got and met; null until the comparison is finished, when no thresholds were recorded, and for a report decided under an earlier rule that these bars would not explain: when the user asks why an attempt did not go live, read the bars that are not met back in these words), verdict_checks_withheld (decided_under_earlier_rule when a finished report's bars are withheld because an earlier version of the rule decided it, null otherwise: then say the comparison was decided under an earlier version of the rule, so its bars are not listed -- its verdict stands as recorded; never say they were not recorded, and never work them out from today's rule), cost (training_cents, comparison_cents, scoring_cents, benchmark_cents and total_cents, the attempt's cost_cents on the pipeline; up_to true means comparison_cents and the total are 'up to', the most the comparison machines can have cost), and available_actions — which says what THIS reader may actually do next. Use available_actions rather than guessing from the state: a run can look promotable and not be, because the consent moved or the candidate is already gone.", { run_id: z.string() }, async ({ run_id }) => json(await client.api(`/api/loop/training-runs/${encodeURIComponent(run_id)}`)));
+    server.tool("loop_list_pipelines", "Every pipeline in the workspace: the series of versions it has produced (versions are attempts that were put live; attempts[] lists every attempt, winner or not, each with its outcome, trigger and decision -- rejected means a member chose not to publish it, where outcome says stopped, and expired that nobody decided before the review deadline; auto_rejected that the pipeline's rule turned it down on its own; an expired attempt whose verdict is better was better: never say the rule refused it -- and train_rows and holdout_rows, the samples it trained on and held back; each attempt is judged on at most scored_rows (1,500) of the held-back samples, the same ones every attempt), which one serves the app today (live_version, the same as champion_version, null when what serves is not one of its versions) and whether its deployment still answers (live_deployment.status: serving answers; starting is on its way up; stopped, paused_for_funds or deleted answers nothing and is not billed; paused_by_platform was paused because billing could not be reached and answers nothing; failed answers nothing; live_deployment null means no check is recorded yet or what serves is not a deployment -- never say a live version with any status but serving is answering the app), how each version did against the champion of its day and on the standing benchmark, versions_made against max_versions, the run in flight and which version it will become, and status. Read-only. When the user asks 'is this getting better' or 'what is it doing', answer from this, and read status back honestly: needs_review and needs_funds are waiting on THEM (a decision, a top-up), paused names why in paused_reason (consent_invalid means its settings changed since it was last saved: saving it again starts it), complete means the version limit is reached, and running is the platform working -- including an attempt whose active_run.state is waiting_gpus: it trained, but no GPUs were free to run the new model and the one it is compared with side by side, so both machines were stopped; each try can bill the machine that got a GPU for up to 20 minutes while it waits for the other, nothing bills between tries, it keeps what it trained, tries again after 30 minutes, then 1, 2 and 4 hours, and ends after four tries or 24 hours (its last_reason says when; a last_reason opening 'waiting for room in this workspace to compare' means the workspace is at its limit of running deployments, and stopping a deployment lets it run), and Train now tries again at once. cost_up_to true on an attempt, active_run or a version means its cost_cents is 'up to', the most it can have cost: a comparison machine was charged in it, counted from its first boot while the platform bills it from its first answer (and at the hourly cap when no price was reported), so say 'up to', never 'spent'. next_reason is the pipeline's own sentence about what it is waiting for. scoring_capped_until, when set, is when the workspace's scoring key comes off its monthly cap: until then nothing trains and Train now answers SCORING_CAPPED, so say so instead of offering Train now. month_spent_cents against monthly_ceiling_cents is what decides whether the next run may start; month_spent_cents is what runs started this UTC month have cost at most (a run still in progress counts at its ceilings), so say 'up to', never 'spent'. A pipeline trains only on new samples: it never retrains the conversations an attempt already had with a recipe changed. The list is a page of 100, newest first: total is how many pipelines the workspace has and truncated is true when more exist after this page. When truncated, say it is the newest of total, and pass offset (this page's offset plus its length) for the next page.", { offset: z.number().int().min(0).optional().describe("How many of the newest pipelines to skip") }, async ({ offset }) => json(await client.api("/api/loop/pipelines", {
         params: { offset: offset ? offset.toString() : undefined },
     })));
-    server.tool("loop_get_pipeline", "One pipeline in full, by its id: every version with its verdict, the decision a person made, its win rate and benchmark score, what it cost and why it was made (trigger, in the same words as attempts[]), plus the run in flight, and what this month's runs have cost at most (month_spent_cents, an upper bound, not an exact spend) against the monthly limit. Read-only: promote, reject and roll back stay on the run tools, and its settings on loop_update_pipeline. minimums says how many samples it has against what the next attempt needs. scoring_capped_until, when set, is when the workspace's scoring key comes off its monthly cap: until then nothing trains and Train now answers SCORING_CAPPED with that time as resumes_at. Only compare benchmark scores whose benchmark_comparable is true; the others were measured on a different yardstick. versions[] are WINNERS ONLY -- attempts that were put live, numbered v1, v2 ... in the order they went live, each with its attempt, base_model_id and model_family -- and live_version is the one serving now; attempts[] is every training run the pipeline made, winner or not, with its outcome (became_version, not_better, inconclusive, stopped, failed, running, waiting_for_review) and trigger (data, schedule, manual, or challenger for the second model trained on the same data). Nothing in flight is a version yet. cost_up_to true on attempts[], active_run or versions[] means that cost_cents is 'up to', the most it can have cost: a comparison machine was charged in it, counted from its first boot while the platform bills from its first answer (and at the hourly cap when no price was reported), so say 'up to', never 'spent'. An active_run.state of waiting_gpus means the attempt trained and is waiting for GPUs to run the new model and the one it is compared with side by side: each try can bill the machine that got a GPU for up to 20 minutes while it waits for the other, nothing bills between tries, it keeps what it trained, tries again after 30 minutes, then 1, 2 and 4 hours, and ends after four tries or 24 hours (its last_reason says when; a last_reason opening 'waiting for room in this workspace to compare' means the workspace is at its limit of running deployments, and stopping a deployment lets it run), and Train now tries again at once.", { rule_id: z.string().describe("The pipeline's id, from loop_list_pipelines") }, async ({ rule_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(rule_id)}`)));
-    server.tool("loop_list_pipeline_benchmarks", "The benchmarks a pipeline is measured on, in priority order (1 is the primary), then the ones it stopped using, and its promotion policy. Every attempt is replayed on each attached benchmark; the policy decides which of those measurements make it a version.", { pipeline_id: z.string() }, async ({ pipeline_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(pipeline_id)}/benchmarks`)));
+    server.tool("loop_get_pipeline", "One pipeline in full, by its id: every version with its verdict, the decision a person made, its win rate, its pass rates on the held-back samples (candidate_pass_rate and incumbent_pass_rate: the share each side passed, score 0.70 or more) and benchmark score, what it cost and why it was made (trigger, in the same words as attempts[]), plus the run in flight, and what this month's runs have cost at most (month_spent_cents, an upper bound, not an exact spend) against the monthly limit. Read-only: promote, reject and roll back stay on the run tools, and its settings on loop_update_pipeline. minimums says how many samples it has against what the next attempt needs. scoring_capped_until, when set, is when the workspace's scoring key comes off its monthly cap: until then nothing trains and Train now answers SCORING_CAPPED with that time as resumes_at. Only compare benchmark scores whose benchmark_comparable is true; the others were measured on a different yardstick. versions[] are WINNERS ONLY -- attempts that were put live, numbered v1, v2 ... in the order they went live, each with its attempt, base_model_id and model_family -- and live_version is the one the app's serving name reaches now, with live_deployment.status saying whether its deployment answers (serving answers; starting is on its way up; stopped, paused_for_funds or deleted answers nothing and is not billed; paused_by_platform was paused because billing could not be reached and answers nothing; failed answers nothing; null when no check is recorded yet or what serves is not a deployment); attempts[] is every training run the pipeline made, winner or not, with its outcome (became_version, not_better, inconclusive, stopped, failed, running, waiting_for_review), trigger (data, schedule, manual, or challenger for the second model trained on the same data), decision (what was decided about it once compared, and by whom: promoted, auto_promoted, rejected, auto_rejected, expired or rolled_back; null while nothing has been -- rejected means a member chose not to publish it, where outcome says stopped, so say a member declined it rather than that it stopped; promoted that a member put it live and auto_promoted that the pipeline's rule did; auto_rejected that the pipeline's rule turned it down on its own; expired that nobody decided before the review deadline, so it was retired undecided -- an expired attempt whose verdict is better reads stopped and was better: never say the rule refused it), and train_rows and holdout_rows (the samples its set trained on and held back; null until its set was built). Nothing in flight is a version yet. holdout is the pipeline's held-back share: percent (5 to 20), min_rows (at least that many samples are held back whatever the share), rows (how many it holds back today) and held_before (how many of rows an earlier run of the pipeline held back: they stay held back whatever became of that run and whatever the share says, so a lower share releases none of them; before the first version the first attempt needs held_before plus 1,000 to train on when that is more than the share asks for). Each attempt is judged on at most scored_rows (1,500) of the held-back samples, the same ones every attempt, so rows above that only keeps samples out of training. storage is what the pipeline's history takes up in storage (bytes, objects, and where: platform, your_storage or both): every conversation, answer and scoring reason of its reports, and the training rows its attempts are built from, is kept there and never in the database; it is not billed, counts toward the workspace's storage quota and is deleted with the pipeline. An active_run whose last_reason says it is waiting for storage is holding answers already paid for until storage can keep them; they are kept then without asking the models again. cost_up_to true on attempts[], active_run or versions[] means that cost_cents is 'up to', the most it can have cost: a comparison machine was charged in it, counted from its first boot while the platform bills from its first answer (and at the hourly cap when no price was reported), so say 'up to', never 'spent'. An active_run.state of waiting_gpus means the attempt trained and is waiting for GPUs to run the new model and the one it is compared with side by side: each try can bill the machine that got a GPU for up to 20 minutes while it waits for the other, nothing bills between tries, it keeps what it trained, tries again after 30 minutes, then 1, 2 and 4 hours, and ends after four tries or 24 hours (its last_reason says when; a last_reason opening 'waiting for room in this workspace to compare' means the workspace is at its limit of running deployments, and stopping a deployment lets it run), and Train now tries again at once.", { rule_id: z.string().describe("The pipeline's id, from loop_list_pipelines") }, async ({ rule_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(rule_id)}`)));
+    server.tool("loop_list_pipeline_benchmarks", "The benchmarks a pipeline is measured on, in priority order (1 is the primary), then the ones it stopped using, and its promotion policy. Every attempt is replayed on each attached benchmark; the policy decides which of those measurements make it a version. Each says how it scores an answer (check: judge, a scoring model reading its rubric, or exact, the answer must match its right answer) and what its pinned set takes up in storage (storage: bytes, objects, where), so no benchmark needs reading one by one to say either.", { pipeline_id: z.string() }, async ({ pipeline_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(pipeline_id)}/benchmarks`)));
     server.tool("loop_attach_pipeline_benchmark", "Measure a pipeline on one more benchmark (from loop_list_benchmarks or loop_create_benchmark), last unless priority is given; priority 1 makes it the primary. Applies from the next attempt; older versions are not re-measured and show not_measured for it. 409 BENCHMARK_ALREADY_ATTACHED or BENCHMARK_RETIRED.", {
         pipeline_id: z.string(),
         benchmark_id: z.string(),
@@ -3195,13 +3233,23 @@ create (1-5 choices), and the queue itself stays opt-in.`,
     }, async ({ pipeline_id, benchmark_id }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(pipeline_id)}/benchmarks/${encodeURIComponent(benchmark_id)}`, {
         method: "DELETE",
     })));
-    server.tool("loop_compare_versions", "A pipeline's versions side by side: each version's head-to-head result and its score on every benchmark, with comparable flags. Pass a and b (VERSION numbers, e.g. 1 and 5) for those two and a difference per benchmark; a benchmark attached after a version is not_measured for it, never a zero. Compare only scores marked comparable.", {
+    server.tool("loop_compare_versions", "A pipeline's versions side by side: each version's head-to-head result and its score on every benchmark, each with the share of conversations each side passed (incumbent_pass_rate, candidate_pass_rate; score 0.70 or more), with comparable flags. Pass a and b (VERSION numbers, e.g. 1 and 5) for those two and a difference per benchmark; a benchmark attached after a version is not_measured for it, never a zero. Each benchmark says how it scores an answer (check: judge or exact). Compare only scores marked comparable.", {
         pipeline_id: z.string(),
         a: z.number().int().min(1).optional().describe("A version number"),
         b: z.number().int().min(1).optional().describe("Another version number"),
     }, async ({ pipeline_id, a, b }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(pipeline_id)}/versions/compare`, {
         params: { a: a?.toString(), b: b?.toString() },
     })));
+    server.tool("loop_compare_attempts", "Any two attempts of a pipeline side by side, put live or not -- the one to use for 'why was attempt 8 not better than 7' or a challenger against the attempt whose samples it trained on. a and b are RUN ids (attempts[].run_id from loop_get_pipeline), two different attempts of this pipeline. Each side has its outcome, decision and held-back numbers against what it was compared with on its day, with the share each side passed (score 0.70 or more); never subtract two attempts' win rates, their opponents differ. benchmarks lists every benchmark both replayed, with delta (b minus a) only where comparable, why otherwise. shared_sample counts the held-back conversations both were scored on and how many b scored higher, lower or the same on (by the tie band: 0.02 or more above counts as higher; less than 0.02 apart is about the same): these are two absolute scores, not a reading of one answer against the other. shared_sample.a and .b are each attempt's mean and pass rate over those same conversations -- the like-for-like figures, which may be subtracted; each side's holdout numbers are over its own held-back set, which grows between attempts, so do not subtract those. When shared_sample.comparable is false the two were scored differently (another scoring model, holdout.judge_model, or another rubric) or how one was scored was not recorded (an older attempt that froze no rubric is never taken to match): say why in its words, and never read a direction off them. A conversation whose right answer changed between the two attempts has direction null and says why (answer_changed counts them); so does one where it is not known whether both were scored against the same right answer, because one was compared before that was recorded (answer_unknown counts them; never read those as unchanged or as changed). items is one page of them, numbers and ids only: read an answer with loop_list_evaluation_items on that side's holdout.evaluation_id. filter narrows items and total to one direction, and is refused when the two are not comparable.", {
+        pipeline_id: z.string(),
+        a: z.string().describe("The first attempt's run id"),
+        b: z.string().describe("The second attempt's run id; directions say b against a"),
+        filter: z.enum(["higher", "lower", "same"]).optional().describe("Only the conversations where b scored higher, lower, or the same"),
+        limit: z.number().int().min(1).max(200).optional().describe("Conversations per page, 50 when absent"),
+        offset: z.number().int().min(0).optional(),
+    }, async ({ pipeline_id, a, b, filter, limit, offset }) => json(await client.api(`/api/loop/pipelines/${encodeURIComponent(pipeline_id)}/attempts/compare`, {
+        params: { a, b, filter, limit: limit?.toString(), offset: offset ? offset.toString() : undefined },
+    })));
     server.tool("loop_promote_training_run", "Re-point the serving name at the candidate: from this moment the workspace's traffic is answered by the newly trained model. Read the comparison back to the user first (loop_get_evaluation) and get an explicit yes — this changes what real customers get, and it is only undoable for 30 days with loop_rollback_training_run. An INCONCLUSIVE comparison is refused unless force is set, and forcing one means promoting a model nothing showed to be better; say that out loud rather than setting force to get past the error.", {
         run_id: z.string(),
         expected_revision: z.number().int().min(0).optional().describe("The pipeline revision the decision was made against. Stale answers CONSENT_CHANGED instead of promoting under settings that have since changed."),
@@ -3231,34 +3279,42 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         method: "POST",
         body: { reason },
     })));
-    server.tool("loop_get_evaluation", "The comparison report behind a verdict: how many pairs were scored, the candidate's win rate, the mean scores per rubric dimension, the warnings, and the margins the verdict was measured against. Read the WARNINGS out too, not just the verdict — 'better' from a judge that rarely agrees with this workspace's reviewers, or from a handful of rows, is a number with a caveat attached, and the caveat is in that list.", { evaluation_id: z.string().describe("From the run's links, or loop_get_training_run") }, async ({ evaluation_id }) => json(await client.api(`/api/loop/evaluations/${encodeURIComponent(evaluation_id)}`)));
-    server.tool("loop_list_evaluation_items", "The paired conversations behind the numbers: the same prompt, what the version serving today answered, what the attempt answered, and what the judge made of each. This is how a person checks a verdict instead of trusting it. Filter to winner='serving' to read the pairs the attempt LOST (the items themselves say winner 'incumbent'), which is the honest thing to show somebody before they promote.", {
+    server.tool("loop_get_evaluation", "The comparison report behind a verdict: how many pairs were scored, the candidate's win rate, how many conversations each side passed (score 0.70 or more: candidate_passed, incumbent_passed, and the pass rates over rows_scored, all four null until the report is finished), the mean scores per rubric dimension, the warnings, and the margins the verdict was measured against. Read the WARNINGS out too, not just the verdict — 'better' from a judge that rarely agrees with this workspace's reviewers, or from a handful of rows, is a number with a caveat attached, and the caveat is in that list.", { evaluation_id: z.string().describe("From the run's links, or loop_get_training_run") }, async ({ evaluation_id }) => json(await client.api(`/api/loop/evaluations/${encodeURIComponent(evaluation_id)}`)));
+    server.tool("loop_list_evaluation_items", "The paired conversations behind the numbers: the same prompt, the reference a person gave for it (answer, kind and the tool_calls it makes; null when none), what the version serving today answered, what the attempt answered, each with the tool_calls it made, what the judge made of each (judge.overall, judge.dimensions and judge.reason, its reason in its own words) and whether each passed (score 0.70 or more), and pairwise: the direct comparison asked in both orders, each with its reason, where a win counts only when both orders agree. This is how a person checks a verdict instead of trusting it. Filter to winner='serving' to read the pairs the attempt LOST (the items themselves say winner 'incumbent'), which is the honest thing to show somebody before they promote.", {
         evaluation_id: z.string(),
         winner: z.enum(["candidate", "serving", "tie"]).optional().describe("Only pairs with this outcome: candidate (the attempt won), serving (the version serving today won) or tie"),
+        candidate_passed: z.boolean().optional().describe("true: only pairs where the attempt's answer passed (score 0.70 or more); false: only pairs where it was scored below 0.70. With incumbent_passed it narrows to one cell: candidate_passed false and incumbent_passed true are the regressions. A pair that could not be scored matches neither"),
+        incumbent_passed: z.boolean().optional().describe("true: only pairs where the serving version's answer passed (score 0.70 or more); false: only pairs where it was scored below 0.70"),
+        orders: z.enum(["agreed", "disagreed"]).optional().describe("Only pairs whose two side-by-side orders picked the same side (agreed), or did not"),
+        has_reference: z.boolean().optional().describe("Only pairs with (true) or without (false) a right answer"),
+        status: z.enum(["scored", "failed"]).optional(),
+        sort: z.enum(["ordinal", "score_change"]).optional().describe("score_change: the attempt's score less the other's, largest first"),
+        q: z.string().min(2).max(200).optional().describe("Search the conversation, the right answer, both answers and every reason (case-insensitive); searched in the answer says how many pairs were read. A search can be refused 429 RATE_LIMITED while this workspace already has a search running or while the service is running as many searches as it can at once, so a 429 does not say which: never tell the user they already had one running; try again after the seconds its message gives"),
         limit: z.number().int().min(1).max(200).optional().describe("50 when omitted, at most 200"),
         offset: z.number().int().min(0).optional(),
-    }, async ({ evaluation_id, winner, limit, offset }) => json(await client.api(`/api/loop/evaluations/${encodeURIComponent(evaluation_id)}/items`, {
-        params: { winner, limit: limit?.toString(), offset: offset?.toString() },
+    }, async ({ evaluation_id, winner, candidate_passed, incumbent_passed, orders, has_reference, status, sort, q, limit, offset }) => json(await client.api(`/api/loop/evaluations/${encodeURIComponent(evaluation_id)}/items`, {
+        params: { winner, candidate_passed: candidate_passed?.toString(), incumbent_passed: incumbent_passed?.toString(), orders, has_reference: has_reference?.toString(), status, sort, q, limit: limit?.toString(), offset: offset?.toString() },
     })));
-    server.tool("loop_create_benchmark", "Pin a fixed set of 20 to 200 conversations (source.kind 'traces' with trace_ids) as a benchmark: the same questions and the same judge, replayed on every attempt of every pipeline it is attached to, so 'is it getting better' has a yardstick that does not move. It is refused rather than trimmed when a conversation has nothing to ask a model. Attach it with loop_attach_pipeline_benchmark.", {
+    server.tool("loop_create_benchmark", "Pin a fixed set of 20 to 200 conversations (source.kind 'traces' with trace_ids) as a benchmark: the same questions, each with the right answer a person gave it, scored the same way on every attempt of every pipeline it is attached to, so 'is it getting better' has a yardstick that does not move. check says how, and is frozen: 'judge' (the default: a scoring model reads both answers against the right answer) or 'exact' (the answer or the tool call must match the right answer, and no model is asked -- for tool calls, numbers and labels; a written answer rarely matches word for word; a value hidden when the conversation was captured, such as an email or a long number kept as [redacted:email], cannot be checked, so an answer that matches everywhere else still scores 0 and says so). An exact benchmark needs a right answer on every conversation: 422 BENCHMARK_ROW_HAS_NO_ANSWER names the ones without and nothing is saved -- mark their answers good or correct them first, leave them out, or use 'judge'. It is refused rather than trimmed when a conversation has nothing to ask a model, or is too large to keep in storage: 422 BENCHMARK_ROW_UNUSABLE names it, and nothing is saved -- leave it out. 409 CONFLICT when the conversations could not be stored just then: nothing was saved; try again in a minute. Attach it with loop_attach_pipeline_benchmark.", {
         name: z.string(),
         source: z.object({
             kind: z.enum(["traces"]).describe("Always 'traces': a benchmark pins the conversations named in trace_ids"),
             trace_ids: z.array(z.string()).min(1).describe("The conversations to pin, 20 to 200 (from loop_list_traces, or the trace_ids loop_import_rows returns for a file)"),
         }),
-    }, async ({ name, source }) => json(await client.api("/api/loop/benchmarks", {
+        check: z.enum(["judge", "exact"]).optional().describe("How answers are scored: judge (the default: a scoring model reads both answers against the right answer) or exact (the answer or the tool call must match the right answer; no model is asked)"),
+    }, async ({ name, source, check }) => json(await client.api("/api/loop/benchmarks", {
         method: "POST",
-        body: { name, source },
+        body: { name, source, check },
     })));
-    server.tool("loop_list_benchmarks", "The workspace's benchmarks, newest first. status 'active' hides the retired ones.", {
+    server.tool("loop_list_benchmarks", "The workspace's benchmarks, newest first. status 'active' hides the retired ones. Each carries check (how it is scored: judge, a scoring model, or exact, the exact check with no model) and storage -- what its pinned set takes up in storage (bytes, objects, and where: platform, your_storage or both); it is not billed, counts toward the workspace's storage quota and is deleted with the workspace.", {
         status: z.enum(["active", "retired"]).optional(),
         limit: z.number().int().min(1).max(200).optional(),
         offset: z.number().int().min(0).optional(),
     }, async ({ status, limit, offset }) => json(await client.api("/api/loop/benchmarks", {
         params: { status, limit: limit?.toString(), offset: offset?.toString() },
     })));
-    server.tool("loop_get_benchmark", "One benchmark: its frozen judge and scoring rule, and the fingerprint of its set.", { benchmark_id: z.string() }, async ({ benchmark_id }) => json(await client.api(`/api/loop/benchmarks/${encodeURIComponent(benchmark_id)}`)));
-    server.tool("loop_list_benchmark_items", "The conversations a benchmark pinned. A conversation deleted since keeps its pinned copy; its source link reads null.", {
+    server.tool("loop_get_benchmark", "One benchmark: how it is scored (check: judge, a scoring model, or exact, the exact check with no model -- then judge_model is empty), its frozen judge and scoring rule, the fingerprint of its set, and storage -- what its pinned set takes up in storage (bytes, objects, and where: platform, your_storage or both). The set is kept in storage only; it is not billed, counts toward the workspace's storage quota and is deleted with the workspace.", { benchmark_id: z.string() }, async ({ benchmark_id }) => json(await client.api(`/api/loop/benchmarks/${encodeURIComponent(benchmark_id)}`)));
+    server.tool("loop_list_benchmark_items", "The conversations a benchmark pinned, each with the right answer pinned with it (reference: answer, kind and tool_calls -- an answer marked good that only calls a tool has an empty answer and its calls; null when nobody gave one). A conversation deleted since keeps its pinned copy; its source link reads null.", {
         benchmark_id: z.string(),
         limit: z.number().int().min(1).max(200).optional().describe("50 when omitted, at most 200"),
         offset: z.number().int().min(0).optional(),
@@ -3279,7 +3335,21 @@ create (1-5 choices), and the queue itself stays opt-in.`,
         method: "POST",
         body: { reason },
     })));
-    server.tool("loop_get_benchmark_run", "One replay in full: both scores, the difference, the per-dimension breakdown and what it cost. Read status first — a replay that did not score every pinned conversation publishes no score.", { benchmark_run_id: z.string() }, async ({ benchmark_run_id }) => json(await client.api(`/api/loop/benchmark-runs/${encodeURIComponent(benchmark_run_id)}`)));
+    server.tool("loop_get_benchmark_run", "One replay in full: both scores, the difference, the per-dimension breakdown, how many conversations each side passed (score 0.70 or more) and the pass rates, the conversations the attempt won, lost and tied (wins, losses, ties), and what it cost. Read status first — a replay that did not score every pinned conversation publishes no score, and its pass counts, rates and wins are null too. loop_list_benchmark_run_items reads every conversation behind it.", { benchmark_run_id: z.string() }, async ({ benchmark_run_id }) => json(await client.api(`/api/loop/benchmark-runs/${encodeURIComponent(benchmark_run_id)}`)));
+    server.tool("loop_list_benchmark_run_items", "The conversations behind one replay, in the benchmark's order: the pinned prompt and its reference (answer, kind and the tool_calls the right answer makes; null when none), what the version serving today answered, what the attempt answered, each with the tool_calls it made, what the judge made of each (judge.overall, judge.dimensions and judge.reason; on an exact benchmark overall and dimensions.match are 1 or 0 and the reason says what matched or what differed) and whether each passed (score 0.70 or more), and which side won it (winner compares the two scores by the tie band promotion uses: 0.02 or more above counts as higher, less than 0.02 apart is a tie, and it is null until both are scored). Filter to winner='serving' to read the conversations the attempt LOST (the items themselves say winner 'incumbent'). A replay that stopped before it began has no conversations: read its status_reason back instead.", {
+        benchmark_run_id: z.string().describe("The replay's id: from loop_get_benchmark_history, loop_compare_versions, or a run's benchmark_runs"),
+        winner: z.enum(["candidate", "serving", "tie"]).optional().describe("Only conversations with this outcome: candidate (the attempt won), serving (the version serving today won) or tie"),
+        candidate_passed: z.boolean().optional().describe("true: only conversations where the attempt's answer passed (score 0.70 or more); false: only ones where it was scored below 0.70. With incumbent_passed it narrows to one cell: candidate_passed false and incumbent_passed true are the regressions. One that could not be scored matches neither"),
+        incumbent_passed: z.boolean().optional().describe("true: only conversations where the serving version's answer passed (score 0.70 or more); false: only ones where it was scored below 0.70"),
+        has_reference: z.boolean().optional().describe("Only conversations with (true) or without (false) a right answer"),
+        status: z.enum(["scored", "failed"]).optional(),
+        sort: z.enum(["ordinal", "score_change"]).optional().describe("score_change: the attempt's score less the other's, largest first"),
+        q: z.string().min(2).max(200).optional().describe("Search the conversation, the right answer, both answers and every reason (case-insensitive); searched in the answer says how many were read. A search can be refused 429 RATE_LIMITED while this workspace already has a search running or while the service is running as many searches as it can at once, so a 429 does not say which: never tell the user they already had one running; try again after the seconds its message gives"),
+        limit: z.number().int().min(1).max(200).optional().describe("50 when omitted, at most 200"),
+        offset: z.number().int().min(0).optional(),
+    }, async ({ benchmark_run_id, winner, candidate_passed, incumbent_passed, has_reference, status, sort, q, limit, offset }) => json(await client.api(`/api/loop/benchmark-runs/${encodeURIComponent(benchmark_run_id)}/items`, {
+        params: { winner, candidate_passed: candidate_passed?.toString(), incumbent_passed: incumbent_passed?.toString(), has_reference: has_reference?.toString(), status, sort, q, limit: limit?.toString(), offset: offset?.toString() },
+    })));
     server.tool("set_inference_alias", "Point a re-pointable public handle at a deployment. An alias is a NAME customers call that can be moved to different weights without them changing anything — it is what a promotion writes, and what lets a retrained model take over traffic with no client edit. Setting one CHANGES WHAT REAL TRAFFIC REACHES, so confirm with the user first. A name that belongs to a live deployment is refused (ALIAS_NAME_IS_A_DEPLOYMENT) unless the alias is being given that deployment's own name, and a target that is not serving is refused (TARGET_NOT_SERVABLE) rather than pointing the handle at something that cannot answer.", {
         name: z.string().describe("The public handle, e.g. 'support-bot'"),
         target_inference_id: z.string().describe("The deployment the name should reach, from list_inferences"),