runbios-sdk 0.2.1-rc.98 → 0.2.2-dev.171

package/dist/resources/loop.d.ts

@@ -0,0 +1,850 @@
+import type { HttpClient } from '../client.js';
+import type { LoopTrace, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopImportParams, LoopImportResult, LoopSignal, LoopSignalParams, LoopCandidate, LoopCandidateParams, LoopGrader, LoopGraderParams, LoopLabel, LoopLabelCount, LoopGradeResult, LoopDataset, LoopDatasetListParams, LoopDatasetItem, LoopDatasetCreateParams, LoopConfig, LoopConfigParams, LoopBuildRule, LoopBuildRuleParams, LoopBuildRuleUpdateParams, LoopStats, LoopJudge, LoopJudgeParams, LoopAgentCredential, LoopAgentStatus, LoopSampleRun, LoopSampleRunParams, LoopJudgeRun, LoopJudgeRunItems, LoopJudgeRunItemStatus, LoopJudgeWork, LoopJudgeVerdict, LoopJudgeVerdictResult, TrainingRule, TrainingRulePreflight, TrainingRulePreflightRequest, TrainingRuleCreateRequest, TrainingRuleUpdateRequest, TrainingRuleConsentRequest, TrainingRuleListParams, TrainingRuleListResponse, TrainingRuleResponse, TrainingRuleMutationResponse, TrainingRuleDeleteResponse, TrainingRun, TrainingRunListParams, TrainingRunListResponse, TrainingRunResponse, TrainingRunActionResponse, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, Evaluation, EvaluationItemListParams, EvaluationItemsResponse, JudgeAgreement, JudgeAgreementParams, Benchmark, BenchmarkCreateParams, BenchmarkHistoryParams, BenchmarkHistoryResponse, BenchmarkItemListParams, BenchmarkItemsResponse, BenchmarkListParams, BenchmarkListResponse, BenchmarkRetireParams, BenchmarkRun, TrainingRuleBenchmarkRequest, AgentSettings, AgentSettingsRequest } from '../types.js';
+/**
+ * The Conscious Loop -- capture what your model was asked and answered, record
+ * whether it was right, and turn those judgements into training data.
+ *
+ * Nothing is captured until you turn it on for a source, and you can turn it
+ * off again at any time. Anything that looks like a credential or a personal
+ * detail is removed before the record is written, never afterwards.
+ *
+ * Requires an API key carrying `loop:read` and/or `loop:write`. Those scopes are
+ * deliberately absent from the Read Only preset, because what they return is
+ * your raw prompts and completions rather than catalog metadata.
+ *
+ * @example
+ * ```ts
+ * // 1. Decide that this source is recorded.
+ * await client.loop.setConfig('my-agent', { enabled: true, retention_days: 30 });
+ *
+ * // 2. Send what your model was asked and what it answered. Works whoever
+ * //    served it -- us, another provider, or your own servers.
+ * const { trace_id } = await client.loop.capture({
+ *   deployment_id: 'my-agent',
+ *   model: 'gpt-4o',
+ *   messages: [{ role: 'user', content: 'what is our refund window' }],
+ *   completion: 'thirty days',
+ * });
+ *
+ * // 3. Say whether it was right. A rewrite is the most valuable answer here:
+ * //    the model learns your version AND learns to avoid its own.
+ * await client.loop.signal(trace_id, {
+ *   verdict: 'edited',
+ *   correction: 'Thirty days from delivery, no questions asked.',
+ * });
+ *
+ * // 4. Turn the judgements into a training set and take the file.
+ * const ds = await client.loop.createDataset({ name: 'support', method: 'sft' });
+ * const jsonl = await client.loop.downloadDataset(ds.id);
+ * ```
+ */
+export declare class Loop {
+    private readonly _http;
+    /** @internal */
+    constructor(_http: HttpClient);
+    /**
+     * Send one exchange to be captured.
+     *
+     * Works regardless of who served the model: ours, OpenAI, Fireworks, your own
+     * own servers, an agent framework. Pass `tool_calls` and `tools` when the turn
+     * invoked a function -- without them a tool-using exchange trains the model to
+     * answer in prose where it should have called something.
+     *
+     * The returned `trace_id` is OURS. If you pass your own `request_id` it is
+     * kept as an idempotency handle: sending the same one again returns the same
+     * trace rather than storing a second copy, so a retry is safe.
+     *
+     * When the source is not enabled this resolves with `captured: false` and a
+     * reason instead of throwing -- capture must never be the thing that breaks
+     * your application.
+     */
+    capture(params: LoopCaptureParams): Promise<LoopCaptureResult>;
+    /**
+     * Bring data you already have into the loop.
+     *
+     * This does NOT create a training set. It creates conversations, in the same
+     * place captured ones live and subject to the same review, rules, judges and
+     * labels. A row becomes trainable when something says it is good, never
+     * because it arrived in a file -- which is the one guarantee that separates
+     * a corpus from a pile.
+     *
+     * Each row is read for what it actually is. A `prompt`/`chosen`/`rejected`
+     * triple becomes a preference pair; an answer plus a yes-or-no becomes a
+     * thumbs verdict; a question and an answer waits for review; a question with
+     * no answer waits for an answer; a paragraph of prose is refused, because it
+     * is not a conversation.
+     *
+     * A verdict that arrives *with* the file is kept -- discarding somebody's
+     * judgement would be worse -- but it is recorded as having come from your
+     * earlier process rather than from a reviewer here, and the conversation
+     * records that it was imported. Neither fact can be reconstructed later, so
+     * both are written at the door.
+     *
+     * `source` is required and becomes the id every imported conversation is
+     * filed under: "everything in one bucket called import" is a corpus nobody
+     * can slice afterwards. At most 5000 rows per call, because the call is
+     * synchronous and somebody is waiting on it.
+     *
+     * WHAT COMES BACK. `imported` is what this call created and `trace_ids` names
+     * the conversations from the file that are now in the loop, so a caller can
+     * label, review or delete them. `by_shape`, `reviewed` and `needs_review`
+     * count what THIS CALL created, not what was merely recognised and not rows
+     * an earlier import already had, and the sentences in `notes` are written
+     * from those counts, so the numbers and the prose describe the same call and
+     * a re-send cannot report a reviewed conversation as waiting for review.
+     * `refused` is rows whose shape could not be read, which the file fixes;
+     * `not_saved` is rows that were understood and then could not be stored,
+     * which the file cannot fix.
+     *
+     * A call with any `not_saved`, or any `verdicts_not_saved` (the conversation
+     * stored and the judgement that came with it did not), REJECTS with
+     * `LoopImportIncompleteError`, whose `outcome` carries all of the above and
+     * whose `notSavedRows` and `verdictsNotSavedRows` name the positions.
+     *
+     * HOW A RETRY IS SAFE. Every response carries `import_id`, the token this
+     * call was filed under. Send the same rows again with it, and rows that
+     * already arrived come back on `already_present` instead of being stored
+     * twice, while a verdict that failed to write is attempted again. A call that
+     * repeats no token is its own import, deliberately: two calls carrying the
+     * same rows are as likely to be two pages of one export as one call twice,
+     * and guessing "retry" silently drops the second copy of every conversation a
+     * file lists more than once.
+     *
+     * SENDING A FILE IN PAGES. Either give each row its own `row_ids` entry, and
+     * then chunking, ordering and subsets all stop mattering; or send one
+     * `import_id` with the `row_offset` each page starts at. Do not send back
+     * only the rows `notSavedRows` names unless you used `row_ids`: without them
+     * a shorter list moves every row after the gap and imports it again.
+     */
+    importRows(params: LoopImportParams): Promise<LoopImportResult>;
+    /** List captured conversations, newest first. */
+    listTraces(params?: LoopTraceListParams): Promise<LoopTraceListResponse>;
+    /** Read one conversation, with every verdict recorded on it. */
+    getTrace(id: string): Promise<LoopTrace>;
+    /**
+     * Delete a conversation.
+     *
+     * It also leaves every training set built from it, including sets already
+     * exported -- so a customer asking you to delete a conversation genuinely
+     * removes it from what the model will learn next. Training runs that have
+     * already finished are not affected.
+     */
+    deleteTrace(id: string): Promise<{
+        deleted: boolean;
+        trace_id: string;
+    }>;
+    /**
+     * Record a verdict on a conversation.
+     *
+     * Append-only: posting a second verdict does not replace the first. Two
+     * reviewers disagreeing about an answer is information worth keeping.
+     *
+     * `source` says who judged: `human`, `verifier` (a deterministic check --
+     * tests passed, schema valid), `judge` (a model grading a model), or
+     * `behavioural` (what the user did next). When several disagree, a human
+     * outranks a verifier, a verifier outranks a judge, and a judge outranks a
+     * behavioural hint.
+     */
+    signal(traceId: string, params: LoopSignalParams): Promise<LoopSignal>;
+    /**
+     * Thumbs up: this answer was good. Shorthand for `signal(id, { verdict:
+     * 'accepted' })` — there is one stored vocabulary behind both, so a thumbs-up
+     * and an "accepted" can never disagree in your corpus.
+     */
+    upvote(traceId: string, reason?: string): Promise<LoopSignal>;
+    /**
+     * Thumbs down: this answer was bad, and you have nothing better to offer.
+     *
+     * On its own this REMOVES the answer from training rather than teaching
+     * anything — there is no better version to learn. If you know what it should
+     * have said, `correct()` is worth far more.
+     */
+    downvote(traceId: string, reason?: string): Promise<LoopSignal>;
+    /**
+     * The model was wrong; here is what it should have said.
+     *
+     * The most valuable feedback there is, because it produces BOTH halves of a
+     * preference pair from one action: the model learns your answer and learns to
+     * avoid its own.
+     */
+    correct(traceId: string, answer: string, reason?: string): Promise<LoopSignal>;
+    /**
+     * Record the GOLD answer for this question — the reference, regardless of
+     * what the model happened to say.
+     *
+     * Different from `correct()` in a way that matters: a correction asserts the
+     * model was wrong, so the original becomes the rejected half of a pair. A gold
+     * answer asserts nothing about the model, so if it MATCHES what was said, no
+     * preference pair is invented — but SFT still learns it.
+     *
+     * It is the most reusable thing you can record: it trains SFT, forms a DPO
+     * pair when it differs from the answer, and stands in as the GRPO reference
+     * when you have not supplied a separate ground truth.
+     */
+    gold(traceId: string, answer: string, reason?: string): Promise<LoopSignal>;
+    /** Every verdict on one conversation, oldest first. */
+    listSignals(traceId: string): Promise<LoopSignal[]>;
+    /**
+     * Submit an alternative answer to a prompt already captured.
+     *
+     * This is what makes preference learning scale. A human rewrite is the best
+     * signal there is and the least available -- somebody has to sit down and
+     * write it. Sample the model several times for the same prompt, score the
+     * samples, and a DPO pair falls out automatically: best becomes chosen,
+     * worst becomes rejected. Point a stronger model at the prompt instead and
+     * the same machinery does distillation.
+     *
+     * Score them, or they cannot pair: an unscored alternative says nothing about
+     * which answer anybody prefers. `score_source` is required alongside a score,
+     * because precedence between a verifier, a judge and a person is the whole
+     * reason we record who judged.
+     *
+     * A human correction always outranks any score.
+     *
+     * @example
+     * ```ts
+     * for (const sample of await sampleMyModel(prompt, 4)) {
+     *   await client.loop.addCandidate(traceId, {
+     *     completion: sample.text,
+     *     score: await myVerifier(sample.text),
+     *     score_source: 'verifier',
+     *   });
+     * }
+     * ```
+     */
+    addCandidate(traceId: string, params: LoopCandidateParams): Promise<LoopCandidate>;
+    /** Every alternative answer recorded for one prompt, oldest first. */
+    listCandidates(traceId: string): Promise<LoopCandidate[]>;
+    /**
+     * Label a conversation, so it can be selected later.
+     *
+     * An average over everything is the least useful thing to train on. A model
+     * weak at refunds is fixed with refund examples, and you can only select
+     * those if you said so when the conversation arrived.
+     *
+     * A label may name a `parent`, which is what makes a MASTER GROUP: labelling
+     * something `refunds` with parent `billing` makes it selectable as either,
+     * and selecting `billing` later gathers every child without you maintaining
+     * a list of them.
+     *
+     * Labels are lowercased and trimmed, so `Refunds` and `refunds` are one label.
+     */
+    label(traceId: string, labels: string[], parent?: string, attributes?: Record<string, string>): Promise<LoopLabel[]>;
+    /**
+     * Set named dimensions on a conversation: `{ category: 'billing',
+     * language: 'es' }`.
+     *
+     * An upsert per dimension, so correcting `category` leaves `task` and any
+     * bare tags exactly where they were — no delete-then-add.
+     */
+    setAttributes(traceId: string, attributes: Record<string, string>): Promise<LoopLabel[]>;
+    unlabel(traceId: string, label: string, key?: string): Promise<{
+        deleted: boolean;
+        key: string;
+        label: string;
+    }>;
+    /**
+     * Every label in the workspace, with how many conversations carry it and
+     * which master groups those conversations are in.
+     *
+     * TWO FIELDS ANSWER THE GROUP QUESTION, AND THEY ANSWER DIFFERENT ONES.
+     * `parent` is the group ALL of a label's conversations are in, and it is
+     * null the moment they disagree. `parents` is every group ANY of them are
+     * in, with how many of them are in each.
+     *
+     * Build a list of master groups out of `parents`. A label with 12
+     * conversations, 5 of them under `support`, reports `parent: null` and
+     * `parents: [{parent: 'support', traces: 5}]`, so code that reads only
+     * `parent` sees no group at all for it. The registry used to answer
+     * `parent: "support"` for all twelve, which named the group but put seven
+     * conversations in it that nobody had put there.
+     */
+    listLabels(): Promise<LoopLabelCount[]>;
+    /**
+     * Build a training set from the feedback recorded so far.
+     *
+     * The three methods need genuinely different things, so one set cannot be
+     * reshaped into another afterwards:
+     *
+     * - `sft` -- answers you marked right, and answers you rewrote.
+     * - `dpo` -- answers you REWROTE, so a better and a worse version of the same
+     *   reply exist. Nothing else produces a pair.
+     * - `grpo` -- answers with a value or fact they can be checked against, or
+     *   a judge rubric, which scores answers the run has not written yet.
+     * - `kto` -- anything carrying a yes or a no, INCLUDING a thumbs-down with
+     *   nothing written. That row trains nothing under the other three methods,
+     *   which is why this one exists: it is the feedback people actually give.
+     *
+     * `holdout_percent` holds back your most recent work rather than a random
+     * slice, so the evaluation measures whether the model generalised instead of
+     * memorised the same week.
+     *
+     * The result always reports `rejected_counts`: why rows were left out. A
+     * small set with a reason is useful; a small set without one is just alarming.
+     */
+    createDataset(params: LoopDatasetCreateParams): Promise<LoopDataset>;
+    /** List training sets, newest first. */
+    /**
+     * Stand up a rule that builds a set whenever enough new work exists.
+     *
+     * The manual path asks somebody to notice that enough conversations have
+     * been reviewed, remember which filters describe the slice they want, and
+     * press build — every time. A rule is that instruction, stored.
+     *
+     * `spec` is the same selection `createDataset` takes and is replayed
+     * verbatim, so an automatic set is identical to a hand-made one.
+     *
+     * `min_new_rows` counts only work reviewed SINCE THE LAST BUILD. Counting
+     * the whole corpus would fire the rule every interval forever, because a
+     * total that has crossed a threshold stays across it.
+     */
+    createBuildRule(params: LoopBuildRuleParams): Promise<LoopBuildRule>;
+    /**
+     * Every build rule, with what each one last did and why.
+     *
+     * `last_reason` is the field worth reading: a rule quiet because it is
+     * waiting looks exactly like one quiet because it is broken.
+     */
+    listBuildRules(): Promise<LoopBuildRule[]>;
+    /** Stop a standing build rule. Sets it already produced are untouched. */
+    deleteBuildRule(id: string): Promise<{
+        deleted: boolean;
+    }>;
+    listDatasets(params?: LoopDatasetListParams): Promise<LoopDataset[]>;
+    /** Read one training set and its curation report. */
+    getDataset(id: string): Promise<LoopDataset>;
+    /**
+     * Page through the rows a training set actually contains.
+     *
+     * Worth reading before you spend money on a run: each row carries the address
+     * of the conversation it was built from.
+     */
+    listDatasetItems(id: string, params?: {
+        limit?: number;
+        offset?: number;
+    }): Promise<LoopDatasetItem[]>;
+    /**
+     * Download a training set as JSONL -- one training row per line, the format
+     * every trainer in this space reads.
+     *
+     * `split` defaults to the training rows; pass `holdout` for the slice held
+     * back, or `all` for both.
+     */
+    downloadDataset(id: string, split?: 'train' | 'holdout' | 'all'): Promise<string>;
+    /**
+     * Delete a training set.
+     *
+     * The conversations it was built from are untouched -- a set is a selection,
+     * and discarding the selection must not discard the evidence.
+     */
+    deleteDataset(id: string): Promise<{
+        deleted: boolean;
+        dataset_id: string;
+    }>;
+    /**
+     * Write a rule that scores answers without a person.
+     *
+     * Human review is the most trustworthy feedback and the least available. A
+     * grader is written once and applied to every answer afterwards: did it
+     * contain the required phrase, did it parse as the schema you asked for, is
+     * the number within tolerance of the known answer, did it call the function
+     * it should have.
+     *
+     * Every kind here is DETERMINISTIC -- no model call, no network. That is why
+     * a verifier outranks a judge when they disagree: it cannot be flattered and
+     * it cannot drift between runs.
+     *
+     * `weight` is the multiplier: a criterion that matters twice as much gets
+     * twice the weight, and the combined score is the weighted mean over the
+     * rules that actually applied.
+     *
+     * `matches_gold` needs no `expected`: it compares each answer to the gold
+     * answer recorded on that same conversation, or the human correction when
+     * there is no gold, and scores sampled alternatives against the same gold.
+     * A conversation with neither is skipped, not failed, so one rule checks
+     * every labelled question without punishing the unlabelled ones. An
+     * optional `tolerance` widens the match when both sides are bare numbers.
+     *
+     * A caution worth knowing before you write a set: a rule made only of
+     * `forbidden` phrases is satisfied VACUOUSLY by an answer that says nothing.
+     * Pair it with a `required` phrase, or you are rewarding silence.
+     */
+    createGrader(params: LoopGraderParams): Promise<LoopGrader>;
+    /** Every rule this workspace has written. */
+    listGraders(): Promise<LoopGrader[]>;
+    deleteGrader(id: string): Promise<{
+        deleted: boolean;
+        grader_id: string;
+    }>;
+    /**
+     * Apply this workspace's rules to one captured answer AND to every
+     * alternative sampled for it.
+     *
+     * Grading both in one pass is the point: scoring only the original gives a
+     * verdict, while scoring the samples as well gives the preference pair. Sample
+     * your model, call this, and you have DPO data with nobody reading anything.
+     *
+     * A run where no rule could apply writes NOTHING -- no verdict, no scores.
+     * Recording a zero that no rule produced would poison curation with a
+     * judgement nobody reached, so `applied: 0` is reported instead.
+     */
+    grade(traceId: string): Promise<LoopGradeResult>;
+    /** Every source this workspace has configured. */
+    listConfigs(): Promise<LoopConfig[]>;
+    /**
+     * Read the capture setting for one source. A source nobody has configured
+     * reads back as disabled rather than missing, because "we are not recording
+     * this" is the honest answer.
+     */
+    getConfig(source: string): Promise<LoopConfig>;
+    /**
+     * Turn capture on or off for a source, and choose how long it is kept.
+     *
+     * This is the consent decision the whole feature rests on: nothing is
+     * recorded until it is made, and the row remembers who made it and when.
+     * `sample_rate` below 1 records a deterministic fraction, chosen so that every
+     * turn of one conversation is captured or skipped together -- half a
+     * conversation makes training rows with holes in them.
+     */
+    setConfig(source: string, params: LoopConfigParams): Promise<LoopConfig>;
+    /**
+     * How much there is, and how much of it each method could actually use.
+     *
+     * The `ready` numbers are upper bounds: duplicate questions are folded into
+     * one row while a set is built, so the finished count can be lower.
+     */
+    stats(): Promise<LoopStats>;
+    /** Write a rubric: what makes an answer good here, and what to score. */
+    createJudge(params: LoopJudgeParams): Promise<LoopJudge>;
+    listJudges(): Promise<LoopJudge[]>;
+    /**
+     * Rewrite a rubric. Runs already recorded keep the instructions they used:
+     * a verdict has to keep meaning what it meant when it was given.
+     */
+    updateJudge(judgeId: string, params: LoopJudgeParams): Promise<LoopJudge>;
+    /**
+     * Retire a rubric. Its runs go with it; the verdicts they wrote stay. A score
+     * is evidence about an answer, and it does not stop being true because the
+     * rubric was retired.
+     */
+    deleteJudge(judgeId: string): Promise<void>;
+    /**
+     * Open a run: select the conversations now, and freeze the rubric onto them.
+     *
+     * The selection is fixed at this moment on purpose. A run whose selection is
+     * a live query silently grows as conversations arrive, so "we scored the
+     * refunds slice" becomes a claim about a set that no longer exists.
+     */
+    startRun(judgeId: string, limit?: number): Promise<LoopJudgeRun>;
+    listRuns(judgeId?: string): Promise<LoopJudgeRun[]>;
+    getRun(runId: string): Promise<LoopJudgeRun>;
+    /**
+     * Take the next conversations to score, each rendered into a ready-to-send
+     * prompt. Nothing is marked taken, so a caller that dies half way loses
+     * nothing: ask again and the same items come back.
+     */
+    takeWork(runId: string, limit?: number): Promise<LoopJudgeWork>;
+    /**
+     * What happened to each conversation in a run, and why.
+     *
+     * `takeWork` hands out what is still PENDING, so a finished run answers it
+     * with an empty list. This answers with every item and the outcome on it:
+     * `status`, `scored_at`, and `error` — the reason the caller gave for an
+     * item it could not score, which is where a wrong model slug or a refused
+     * key actually shows up. A run that ends "scored 0, failed 3" is otherwise a
+     * number with no detail behind it.
+     *
+     * Pass `status: 'failed'` for the usual question. At most 500 items come
+     * back at a time; when `has_more` is true, call again with the `next_offset`
+     * from the reply.
+     */
+    listRunItems(runId: string, opts?: {
+        status?: LoopJudgeRunItemStatus;
+        limit?: number;
+        offset?: number;
+    }): Promise<LoopJudgeRunItems>;
+    /**
+     * Hand the scores back. A verdict for a conversation outside this run, or
+     * scoring something the rubric never asked for, is refused and reported in
+     * `rejected` rather than silently dropped.
+     *
+     * Pass `finish` to close the run in the same call once you have nothing left
+     * to send. To close a run WITHOUT scores, call `stopRun`: an empty verdict
+     * list with `finish` does the same thing and reads like a mistake.
+     */
+    postVerdicts(runId: string, verdicts: LoopJudgeVerdict[], finish?: boolean): Promise<LoopJudgeVerdictResult>;
+    /**
+     * Close a run that has not finished.
+     *
+     * A judge may have one open run of its own at a time, so an open run BLOCKS
+     * the next one, and the runs that most need closing are the ones nobody can
+     * wait out: a run the agent has parked on an empty balance or a refused key
+     * stays open until the reason is fixed or somebody stops it.
+     *
+     * Nothing is deleted. Every verdict already recorded stays recorded, the
+     * counters keep saying how much of the selection was covered, and the
+     * conversations the run was holding are free for the next one. The run ends
+     * as `stopped` rather than `done`, so an interrupted pass and a completed
+     * one do not read alike.
+     *
+     * A run that finished on its own is not rewritten: stopping one answers 409.
+     *
+     * Returns the stopped run itself, like `startRun` and `getRun`, not the
+     * `{run}` envelope the service sends. Every other single-run method in this
+     * class unwraps, and `Loop.stop_run` in the Python SDK does too, so a
+     * caller who wrote `(await loop.stopRun(id)).status` against one of its
+     * siblings is right here as well.
+     */
+    stopRun(runId: string): Promise<LoopJudgeRun>;
+    /**
+     * Can an agent run here, is one running, and is it on for this workspace.
+     * `available` is about the environment; `online` about the worker;
+     * `credential` about this workspace.
+     */
+    agentStatus(): Promise<LoopAgentStatus>;
+    /**
+     * Turn the agent on: mints the workspace's managed serverless key and
+     * resumes the automatic judges a previous turn-off paused. After this,
+     * automatic judges and sample runs make model calls billed to the
+     * workspace. Idempotent.
+     *
+     * A judge whose model the serving gateway will not route is NOT resumed:
+     * making it automatic would buy a run that fails every conversation. It
+     * stays paused, `judges_still_paused` counts those, and each one carries
+     * `auto_pause_reason` saying so. Point it at a model that is served and
+     * turn the agent on again.
+     *
+     * `monthly_spend_cap_cents` caps what that key may spend on model calls in
+     * a calendar month. Omitted = no cap on a fresh key, and an existing cap is
+     * left as it is; sent while the agent is already on, it moves the cap on
+     * the existing key without minting a new one. When the cap is reached the
+     * agent's model calls are refused until next month and its runs pause with
+     * that reason.
+     */
+    enableAgent(opts?: {
+        monthly_spend_cap_cents?: number;
+    }): Promise<LoopAgentCredential>;
+    /**
+     * Turn the agent off: revokes its key and pauses every automatic judge in
+     * the workspace (`judges_paused`; each judge shows `auto_paused`). Turning
+     * it back on resumes exactly those judges. Open runs stop where they are
+     * and continue if it is turned back on. Nothing already scored or written
+     * is removed.
+     */
+    disableAgent(): Promise<{
+        revoked: boolean;
+        judges_paused: number;
+        message: string;
+    }>;
+    /**
+     * Ask the agent to write `n` alternative answers to each conversation in a
+     * slice with `model`, score each with `judge_id`, and store them as
+     * candidates. This is how preference pairs are made without a person
+     * writing each one: the curation pass pairs the best sample against the
+     * worst wherever the gap is real.
+     *
+     * Cost: up to `n` calls to write plus `n` to judge, per conversation, at the
+     * workspace's serverless rate; `selection.sample` caps the conversations
+     * (max 200) and `n` is capped at 8. Opening a run turns the agent on if it
+     * is off.
+     */
+    createSampleRun(params: LoopSampleRunParams): Promise<LoopSampleRun>;
+    listSampleRuns(): Promise<LoopSampleRun[]>;
+    getSampleRun(runId: string): Promise<LoopSampleRun>;
+    /**
+     * Price a training rule before anyone agrees to it. Writes nothing.
+     *
+     * Takes the create body without `accept_terms` and answers with the estimate
+     * a member has to see first: the pinned model revision, the worst hourly
+     * price each GPU ladder can reach, how many hours each ceiling buys, any
+     * refusals that would stop a create, the API keys that cannot follow a
+     * cutover because they lack `deployments:read`, and `terms_text` -- the
+     * exact sentence to show, with real figures in it.
+     *
+     * Send the `terms_version` it returns back in `createTrainingRule`. Read the
+     * figures out of this response rather than inventing ceilings of your own.
+     *
+     * `valid: false` with an EMPTY `refusals` list is not nothing: check
+     * `unreachable`, which names every peer the platform could not reach. The
+     * rule is savable in that state and would be paused, but the estimate around
+     * it is not trustworthy -- both `worst_hourly_*` are `0`, and so are both
+     * `max_*_hours` beside them -- so do not quote those figures to anyone.
+     * `model_revision` is empty unless training-service actually pinned a
+     * commit; test `unreachable.length > 0` to tell a degraded pass from a
+     * refusal.
+     */
+    preflightTrainingRule(params: TrainingRulePreflightRequest): Promise<TrainingRulePreflight>;
+    /**
+     * Create a training rule and record the consent that pays for it.
+     *
+     * ACCEPTING THE TERMS AUTHORISES SPENDING FROM YOUR WALLET WHILE YOU ARE NOT
+     * PRESENT. From here on the platform may, on its own schedule and without
+     * asking again, build a training set, run a training job on rented GPUs,
+     * book a second deployment to compare against the one serving your traffic,
+     * and pay for the model calls that judge the two. Those charges come out of
+     * the wallet of the member whose credentials make this call, up to the
+     * ceilings in `money`, and they keep recurring for as long as the rule is
+     * enabled. If `promotion.auto_promote` is set, the platform will also
+     * re-point your public handle at the new model with nobody reviewing it.
+     *
+     * `accept_terms` is therefore required, and this method refuses to send the
+     * request without it rather than letting the server decide. Call
+     * {@link preflightTrainingRule} first, show the person whose wallet pays the
+     * `terms_text` and the figures it returns, get an explicit yes, and send the
+     * `terms_version` they were shown. Do not invent ceilings or price caps on
+     * their behalf.
+     *
+     * @example
+     * ```ts
+     * const estimate = await client.loop.preflightTrainingRule(draft);
+     * // show estimate.terms_text and the ceilings to the member, get a yes
+     * const rule = await client.loop.createTrainingRule({
+     *   ...draft,
+     *   accept_terms: { terms_version: estimate.terms_version },
+     * });
+     * ```
+     */
+    createTrainingRule(params: TrainingRuleCreateRequest): Promise<TrainingRule>;
+    /**
+     * Every training rule in the workspace, with what each one last did and why.
+     *
+     * `last_reason` and `paused_reason` are the fields worth reading: a rule
+     * quiet because it is waiting looks exactly like one quiet because its
+     * consent went stale.
+     */
+    listTrainingRules(params?: TrainingRuleListParams): Promise<TrainingRuleListResponse>;
+    /**
+     * Read one rule with its recent runs, what it has spent this month, and how
+     * far its judge agrees with your own reviewers.
+     *
+     * Returned whole rather than unwrapped to the rule: `month_spent_cents` is
+     * the number that says whether the monthly ceiling is about to stop it.
+     */
+    getTrainingRule(id: string): Promise<TrainingRuleResponse>;
+    /**
+     * Edit or pause a rule. Absent means unchanged; a null clears the fields
+     * that can be cleared.
+     *
+     * A money-bearing change -- a ceiling, a model, a GPU ladder, the promotion
+     * policy -- bumps `revision`, clears the recorded consent and STOPS the rule
+     * firing until someone accepts the new amounts. The reply says so in
+     * `consent_required`, and carries a fresh `preflight` with the new figures.
+     * Pass `expected_revision` to be refused with `REVISION_MISMATCH` rather
+     * than overwrite an edit somebody else made in the meantime.
+     */
+    updateTrainingRule(id: string, params: TrainingRuleUpdateRequest): Promise<TrainingRuleMutationResponse>;
+    /**
+     * Accept the rule's current terms, so it may fire again.
+     *
+     * ACCEPTING THE TERMS AUTHORISES SPENDING FROM YOUR WALLET WHILE YOU ARE NOT
+     * PRESENT, on the amounts as they stand right now. This is the same
+     * authorisation {@link createTrainingRule} records, given again because a
+     * money-bearing edit cleared the old one: training, a candidate deployment
+     * and the judge's model calls are charged to the wallet of the member making
+     * this call, up to the rule's ceilings, every time it fires.
+     *
+     * Show the member the current `terms_text` from a fresh
+     * {@link preflightTrainingRule} or from the `preflight` on the update reply,
+     * and send the `revision` those figures belong to. A stale revision is
+     * refused with `409`, which is the point: it means the amounts moved again
+     * after they were read.
+     */
+    consentTrainingRule(id: string, params: TrainingRuleConsentRequest): Promise<TrainingRule>;
+    /**
+     * Fire a rule now, without waiting for its cadence.
+     *
+     * Bypasses the schedule and `min_new_rows` only. The row floors, the money
+     * ceilings and the consent all still apply, so this can answer `409
+     * CONSENT_REQUIRED` or `422 NOT_ENOUGH_ROWS` with the counts it needed.
+     */
+    runTrainingRule(id: string): Promise<TrainingRun>;
+    /**
+     * Delete a rule. An active run is cancelled; runs that already finished, and
+     * anything already promoted, are kept.
+     */
+    deleteTrainingRule(id: string): Promise<TrainingRuleDeleteResponse>;
+    /**
+     * Edit or pause a build rule -- the standing instruction that assembles the
+     * training set a training rule then trains on.
+     *
+     * Changing `spec.deployment_id` while an enabled training rule owns this
+     * build rule is refused with `409`: it would silently retrain the next model
+     * on a different source's conversations.
+     */
+    updateBuildRule(id: string, params: LoopBuildRuleUpdateParams): Promise<LoopBuildRule>;
+    /** Firings, newest first. Filter by rule or by the state they are sitting in. */
+    listTrainingRuns(params?: TrainingRunListParams): Promise<TrainingRunListResponse>;
+    /**
+     * One run with its timeline, the addresses of everything it created, and
+     * what you may do with it right now.
+     *
+     * `timeline` is written in the same transaction as each state change, so it
+     * is the record of what actually happened rather than a reconstruction.
+     * `available_actions` is the honest answer to "can I promote this": a button
+     * that cannot work should never be offered.
+     */
+    getTrainingRun(id: string): Promise<TrainingRunResponse>;
+    /**
+     * Promote the candidate by hand: re-point the public handle at the new model.
+     *
+     * This changes what answers your customers. Pass `expected_revision` to be
+     * refused rather than promote against a consent that moved underneath the
+     * decision, and `force` only to promote a comparison the evaluation called
+     * inconclusive -- without it that case is refused with
+     * `INCONCLUSIVE_REQUIRES_FORCE`.
+     */
+    promoteTrainingRun(id: string, params?: TrainingRunPromoteRequest): Promise<TrainingRunActionResponse>;
+    /** Retire the candidate. What serves your traffic does not change. */
+    rejectTrainingRun(id: string, params?: TrainingRunRejectRequest): Promise<TrainingRunActionResponse>;
+    /**
+     * Undo a promotion, inside the rollback window the run reports in
+     * `rollback_available_until` (30 days).
+     *
+     * The reply carries `serving`: this is the one action besides promotion that
+     * changes what answers your traffic, so what it was put back to is returned
+     * rather than left to be looked up.
+     */
+    rollbackTrainingRun(id: string, params?: TrainingRunRollbackRequest): Promise<TrainingRunActionResponse>;
+    /**
+     * Stop a run that is still moving. Work already paid for is still billed --
+     * cancelling a training job does not refund the hours it burned.
+     */
+    cancelTrainingRun(id: string, params?: TrainingRunCancelRequest): Promise<TrainingRunActionResponse>;
+    /**
+     * The comparison behind a verdict: both models on the same held-out rows,
+     * with identical decoding, judge and grader names resolved.
+     *
+     * Read `warnings` before you read `win_rate`. A win rate over a holdout too
+     * small to mean anything, or one measured by a judge that disagrees with
+     * your own reviewers, is reported with the warning that says so rather than
+     * withheld. `margin_used` is the thresholds this verdict was measured
+     * against, frozen with the report, so a later policy change cannot rewrite
+     * what a past decision meant.
+     */
+    getEvaluation(id: string): Promise<Evaluation>;
+    /**
+     * The paired conversations behind the numbers: one prompt, both answers, the
+     * scores each earned, and which won.
+     *
+     * Filter by `winner` to read the losses first, which is where a verdict is
+     * actually checked. `limit` is capped at 100 by the service.
+     */
+    listEvaluationItems(id: string, params?: EvaluationItemListParams): Promise<EvaluationItemsResponse>;
+    /**
+     * How far a judge agrees with your own reviewers, over the conversations
+     * both have scored.
+     *
+     * This is a gate, not a badge: a rule's `min_judge_agreement` refuses to
+     * promote on the word of a judge that does not agree with the people whose
+     * product it is. `enough_pairs` is the field to read first -- "not enough
+     * reviewer overlap yet" is an answer, and 100% of two pairs is not.
+     */
+    getJudgeAgreement(judgeId: string, params?: JudgeAgreementParams): Promise<JudgeAgreement>;
+    /**
+     * Pin a fixed set of conversations, with a judge frozen beside them, and
+     * measure every future run against it.
+     *
+     * The comparison answers "is this candidate better than what serves today".
+     * It cannot answer "is my model getting better", because its rows, its
+     * opponent and its judge all move between runs. A benchmark is the other
+     * instrument: the same conversations, the same oracle, the same decoding,
+     * replayed against both models on every run that finishes its comparison,
+     * and reported as two absolute numbers on a scale that does not move.
+     *
+     * 10 to 200 conversations. A named conversation with nothing to ask a model
+     * is refused rather than skipped, because pinning 47 of the 50 you chose is
+     * the set being wrong from the first day and you would never find out.
+     *
+     * It raises no amount you have already agreed to. The replay's calls come
+     * out of the rule's existing `eval_ceiling_cents`, and
+     * `per_run_ceiling_cents` can only lower what is spent inside that.
+     */
+    createBenchmark(params: BenchmarkCreateParams): Promise<Benchmark>;
+    /** Your benchmarks, newest first. Filter by `status` to hide retired ones. */
+    listBenchmarks(params?: BenchmarkListParams): Promise<BenchmarkListResponse>;
+    /**
+     * One benchmark: the frozen oracle, the scoring rule and the fingerprint of
+     * the set.
+     *
+     * `items_digest` is recomputed from the rows and compared at the start of
+     * every replay, so "the set cannot drift" is something you can check rather
+     * than something the platform promises.
+     */
+    getBenchmark(id: string): Promise<Benchmark>;
+    /**
+     * The pinned conversations, paged. `limit` is capped at 100 by the service.
+     *
+     * `source_trace_id` and `source_trace_url` come back null once the
+     * conversation a row was copied from has been deleted. The row itself stays
+     * and every number already measured against it stays exactly as comparable
+     * as it was: retention cannot shrink a benchmark.
+     */
+    listBenchmarkItems(id: string, params?: BenchmarkItemListParams): Promise<BenchmarkItemsResponse>;
+    /**
+     * The trend: every score this benchmark has produced, newest first, each
+     * beside the checkpoint that produced it and what you then did about it.
+     *
+     * EVERY REPLAY IS A POINT, including the ones with no numbers on them. A
+     * series that silently dropped the runs where the benchmark could not be
+     * scored would read as an unbroken line and not be one, so those points come
+     * back with `candidate_score` null and `status_reason` saying why.
+     */
+    getBenchmarkHistory(id: string, params?: BenchmarkHistoryParams): Promise<BenchmarkHistoryResponse>;
+    /**
+     * Stop replaying a benchmark, and detach it from every rule that names it.
+     *
+     * Nothing measured is removed. There is no delete and no update on purpose:
+     * the series of numbers measured against a set is what a benchmark is for,
+     * so an edit would make everything before it incomparable with everything
+     * after it and a delete would throw the series away. Changing the set means
+     * pinning a new benchmark, and retiring the old one releases its name.
+     *
+     * Retiring twice is somebody pressing a button twice: the second call
+     * answers with the retired benchmark rather than a refusal.
+     */
+    retireBenchmark(id: string, params?: BenchmarkRetireParams): Promise<Benchmark>;
+    /**
+     * One replay in full: both absolute scores, the difference between them on
+     * the same set, the per-dimension breakdown and what it cost.
+     *
+     * Read `status` before you read the scores. A replay that did not score
+     * every pinned conversation publishes no score at all -- all three of
+     * `candidate_score`, `incumbent_score` and `score_delta` are null and
+     * `status_reason` is the sentence that explains it -- because a mean over
+     * whichever conversations happened to succeed is a measurement of a
+     * different set, which is the exact defect a standing benchmark removes.
+     */
+    getBenchmarkRun(id: string): Promise<BenchmarkRun>;
+    /**
+     * Attach a benchmark to a rule, or send `{ benchmark_id: null }` to detach
+     * it.
+     *
+     * Its own route rather than a field on the rule body, because it is a
+     * decision to replay a fixed set on every future run of this rule for as
+     * long as it stands, and its refusals -- retired, or belonging to another
+     * workspace -- are about the benchmark rather than about the rule.
+     *
+     * It does not invalidate consent and the reply says so: attaching raises
+     * neither the amount set aside for judge calls nor the amount set aside for
+     * keeping the new model available, so nobody is asked to read the same
+     * sentence again. A retired benchmark is refused with `BENCHMARK_RETIRED`,
+     * because a rule pointed at one would report no number on every run and say
+     * nothing about why.
+     */
+    setTrainingRuleBenchmark(id: string, params: TrainingRuleBenchmarkRequest): Promise<TrainingRuleMutationResponse>;
+    /**
+     * The workspace's agent options: which model it defaults to, the system
+     * prompts it judges and samples with, and the monthly cap on what its model
+     * calls may spend.
+     */
+    getAgentSettings(): Promise<AgentSettings>;
+    /**
+     * Change them. An absent key leaves that setting exactly where it is; a
+     * present null returns it to the platform default. Those are three
+     * instructions, not two, so `{}` changes nothing.
+     *
+     * `eval_monthly_cap_cents` is pushed to the workspace's managed key, so it
+     * caps what the agent can spend even if a rule's own ceilings are higher.
+     */
+    updateAgentSettings(params: AgentSettingsRequest): Promise<AgentSettings>;
+}