runbios-sdk 0.3.0 → 0.4.0-dev.333

package/dist/types.d.ts

@@ -8,13 +8,13 @@ export interface BiOSConfig {
     orgId?: string;
     /** Workspace ID. Optional when using API keys (resolved from the key). Can override for multi-workspace keys. */
     workspaceId?: string;
-    /** Base URL for the API. Falls back to the RUNBIOS_BASE_URL environment variable (legacy: BIOS_BASE_URL), then the canonical https://api.runbios.ai hostname. */
+    /** Base URL for the API. Falls back to the RUNBIOS_BASE_URL environment variable (legacy: BIOS_BASE_URL), then the canonical https://api-dev.runbios.ai hostname. */
     baseUrl?: string;
     /** Request timeout in milliseconds. Defaults to 30000. */
     timeout?: number;
     /** Default per-deployment inference key. Can be overridden per inference call. */
     inferenceKey?: string;
-    /** Inference base URL. Defaults to baseUrl, then https://api.runbios.ai. */
+    /** Inference base URL. Defaults to baseUrl, then https://api-dev.runbios.ai. */
     inferenceBaseUrl?: string;
     /** End-to-end inference timeout in milliseconds. Defaults to 15 minutes. */
     inferenceTimeout?: number;
@@ -2439,7 +2439,10 @@ export type LoopSource = 'human' | 'verifier' | 'judge' | 'behavioural';
  * trained on in place of the model's.
  * `gold` records the reference answer for the question, making no claim about
  * whether the model was right; it is trained on the same way.
- * `scored` is a score the platform's own scoring wrote; it is read, never sent.
+ * `scored` is the judge's score of a sample nobody reviewed (source `judge`),
+ * written by the platform's automatic scoring; it is read, never sent. It is
+ * not feedback: the sample stays unreviewed, and it is trained on only when
+ * the judge picked it (`picked`, a score of 0.8 or more).
  */
 export type LoopVerdict = 'accepted' | 'rejected' | 'edited' | 'scored' | 'gold';
 /** The verdicts feedback can send: a person's good, bad, correction or reference answer. */
@@ -2665,7 +2668,14 @@ export interface LoopImportResult {
      * was written.
      */
     by_shape: Record<string, number>;
+    /** The refused rows counted by code. Kept for older readers: show `refused_reasons`, which carries the words. */
     refused_why: Record<string, number>;
+    /**
+     * Why rows were refused, one entry per reason: its `code`, how many rows
+     * (`count`) and one plain `sentence` to show. Absent from a service from
+     * before it was sent; `refused_why` then has the codes alone.
+     */
+    refused_reasons?: LoopImportRefusal[];
     /**
      * The conversations from this file that are now in the loop, in file order:
      * the ones this call created and the ones it found already there. Without
@@ -2675,6 +2685,12 @@ export interface LoopImportResult {
     trace_ids?: string[];
     notes: string[];
 }
+/** One reason an import refused rows: the code, how many rows, and the sentence to show for it. */
+export interface LoopImportRefusal {
+    code: string;
+    count: number;
+    sentence: string;
+}
 export interface LoopSignal {
     id: string;
     trace_id: string;
@@ -2688,6 +2704,18 @@ export interface LoopSignal {
     /** What the caller attached when recording it, or null. */
     metadata: Record<string, unknown> | null;
     created_at: string;
+    /**
+     * Present only on the judge's score of an unreviewed sample (source
+     * `judge`, verdict `scored`): true when the score reached the bar (0.8) and
+     * the answer is trained on, false when it did not. No feedback route writes
+     * it, and a person's feedback on the sample decides over it.
+     */
+    picked?: boolean;
+    /**
+     * True on the judge's score when its reason is kept in storage and could
+     * not be read just now: `reason` is then null. Read the sample again later.
+     */
+    reason_unavailable?: boolean;
 }
 /** One feedback item for `loop.signalMany`: the sample's id and its feedback. */
 export interface LoopSignalManyItem extends LoopSignalParams {
@@ -2776,10 +2804,24 @@ export interface LoopTrace {
     /** Who gave that verdict. */
     verdict_source?: LoopSource;
     /**
-     * The score a `scored` verdict carries: above zero it trains the answer as
-     * it stands (good feedback), zero or below it trains nothing (bad).
+     * The score a `scored` verdict carries: the judge's score of a sample nobody
+     * reviewed. Whether it is trained on is `judge.picked`, never this number.
      */
     verdict_score?: number;
+    /**
+     * The judge's score of this sample, when the platform's automatic scoring
+     * scored it (it scores only samples nobody reviewed): `picked` is true when
+     * the score reached the bar (0.8) and the answer is trained on. Present
+     * whatever a person said later; a person's feedback decides over it.
+     * Absent when the judge never scored it.
+     */
+    judge?: LoopTraceJudge;
+    /**
+     * What happens to this sample in the pipeline the list was narrowed to
+     * (`pipeline`). Only on the rows of a list read with `pipeline`; absent from
+     * a service from before it was sent.
+     */
+    state?: LoopSampleState;
     /** Its tags. On the single read only. */
     labels?: LoopLabel[];
     /**
@@ -2795,12 +2837,49 @@ export interface LoopTrace {
      */
     payload_unavailable?: boolean;
 }
+/** The judge's score of an unreviewed sample: its mean over the rubric's aspects, and whether it reached the bar. */
+export interface LoopTraceJudge {
+    score: number;
+    /** True at 0.8 or more: the answer is trained on, as long as no person says otherwise. */
+    picked: boolean;
+}
+/**
+ * What happens to a sample in one pipeline, in that pipeline's terms:
+ * - `held_back`: held back to test its attempts, never trained on;
+ * - `learned`: a version that went live learned it (`version_no`);
+ * - `trained`: an attempt trained on it that did not go live (`attempt_no`);
+ * - `waiting`: it waits for the next attempt;
+ * - `left_out`: the pipeline does not train on it as it is, and `reason`
+ *   says why.
+ *
+ * `reason` is always the plain sentence to show.
+ */
+export type LoopSampleStateName = 'held_back' | 'learned' | 'trained' | 'waiting' | 'left_out';
+/** A sample's state in one pipeline: see {@link LoopSampleStateName}. */
+export interface LoopSampleState {
+    state: LoopSampleStateName;
+    /** The attempt it is about (the one that learned it, held it back or trained on it); null when there is none. */
+    attempt_no: number | null;
+    /** The version that learned it; null unless `state` is `learned`. */
+    version_no: number | null;
+    /** The plain sentence that says it (for `left_out`, why): show it as it is sent. */
+    reason: string;
+}
+/**
+ * The judge's split of the samples nobody reviewed: `picked` (scored 0.8 or
+ * more, trained on), `below_bar` (scored lower, not trained on) and
+ * `unscored` (not scored yet).
+ */
+export type LoopJudgeFilter = 'picked' | 'below_bar' | 'unscored';
 export interface LoopTraceListParams {
     deployment_id?: string;
     conversation_id?: string;
     from?: string;
     to?: string;
-    /** Only conversations that already carry a verdict. */
+    /**
+     * Only conversations a person gave feedback on. The judge's score of an
+     * unreviewed sample is not feedback: see `judge`.
+     */
     signalled?: boolean;
     /** One bare tag. A tag with children matches them too. */
     label?: string;
@@ -2818,17 +2897,42 @@ export interface LoopTraceListParams {
     /** `captured` (our model's own behaviour) or `imported` (a file somebody brought). */
     origin?: 'captured' | 'imported';
     /**
-     * Only the conversations one pipeline's selection covers -- any of its tags
-     * -- reviewed or not. The other filters narrow it further.
+     * Only the conversations one pipeline trains on -- any of its tags or its
+     * datasets, narrowed by its conditions -- reviewed or not. Each row then
+     * carries `state`, what happens to it in that pipeline. The other filters
+     * narrow it further. Not with `dataset`.
      */
     pipeline?: string;
     /**
-     * Only the conversations whose deciding verdict is this feedback: `good`
-     * (accepted), `bad` (rejected) or `corrected` (edited, or gold).
+     * Only the conversations one saved dataset selects (its id): the same
+     * samples, and the same `total`, as its `samples` count. Not with
+     * `pipeline`.
+     */
+    dataset?: string;
+    /**
+     * Only the samples of one attempt's set (a run id): with `split`, the ones
+     * it trained on (`train`) or held back to test it (`held_back`).
+     */
+    attempt?: string;
+    /** With `attempt`: `train` or `held_back`. */
+    split?: 'train' | 'held_back';
+    /**
+     * Only the conversations whose deciding feedback from a person is this:
+     * `good` (accepted), `bad` (rejected) or `corrected` (edited, or gold).
+     * The judge's score is never one of these words.
      */
     signal?: 'good' | 'bad' | 'corrected';
-    /** Only the conversations nobody has given feedback on yet. */
+    /**
+     * Only the conversations no person has given feedback on yet, whether or
+     * not the judge scored them.
+     */
     unsignalled?: boolean;
+    /**
+     * The unreviewed samples, split by the judge: `picked`, `below_bar` or
+     * `unscored` (see {@link LoopJudgeFilter}). Each implies no feedback from a
+     * person; the three add up to `unsignalled`.
+     */
+    judge?: LoopJudgeFilter;
     /**
      * Text search: 2 to 200 characters, case-insensitive, matched as written
      * (`%` and `_` are plain characters) in every message, the answer and the
@@ -2891,6 +2995,29 @@ export interface LoopStats {
      * were not recorded under a tag, most first. Empty when every one was.
      */
     capture: LoopCaptureHealth;
+    /** How many samples were recorded in the last 48 hours. Absent from a service from before it was sent. */
+    samples_recent?: number;
+    /**
+     * Every sample by the feedback a person gave it, in the Samples list's
+     * words (`unreviewed`: no feedback from a person yet). They add up to
+     * `traces`. Absent from a service from before it was sent.
+     */
+    feedback?: LoopFeedbackCounts;
+    /**
+     * How many pipelines the workspace has, and how many of them are training,
+     * waiting on you (`needs_you`), serving or paused: the counts of
+     * `listPipelines({ status })`. Absent from a service from before it was
+     * sent.
+     */
+    pipelines?: LoopPipelineCounts;
+}
+/** `LoopStats.pipelines`: the workspace's pipelines, counted by what they are doing. */
+export interface LoopPipelineCounts {
+    total: number;
+    training: number;
+    needs_you: number;
+    serving: number;
+    paused: number;
 }
 /** `LoopStats.capture` and `Pipeline.capture`: what the one-header capture did not record. */
 export interface LoopCaptureHealth {
@@ -2921,9 +3048,21 @@ export interface LoopTracePipeline {
     name: string;
     /** Whether the pipeline is switched on. A paused one still selects the conversation. */
     enabled: boolean;
+    /**
+     * What happens to the conversation in this pipeline (see
+     * {@link LoopSampleStateName}). Absent from a service from before it was
+     * sent.
+     */
+    state?: LoopSampleStateName;
+    /** See {@link LoopSampleState.attempt_no}. */
+    attempt_no?: number | null;
+    /** See {@link LoopSampleState.version_no}. */
+    version_no?: number | null;
+    /** See {@link LoopSampleState.reason}. */
+    reason?: string;
 }
 export interface LoopTracePipelinesResponse {
-    /** Live pipelines whose tags select this conversation. Empty is a real answer. */
+    /** Live pipelines whose tags, datasets and conditions select this conversation. Empty is a real answer. */
     pipelines: LoopTracePipeline[];
     /** Being selected is not being trained on; this sentence says so. */
     note: string;
@@ -2953,6 +3092,8 @@ export interface LoopMetricPoint {
     billed_candidate_cents: number;
     spent_eval_cents: number;
     benchmark_spent_cents: number;
+    /** Scoring the samples nobody reviewed, in the passes the attempt took when it was booked. */
+    sample_scoring_cents?: number;
     evaluation_id: string | null;
     rows_scored: number | null;
     rows_failed: number | null;
@@ -2967,9 +3108,22 @@ export interface LoopMetricPoint {
     judge_candidate_mean: number | null;
     /** The trainer's loss on its validation slice, as {@link Evaluation.trainer_eval_loss}. */
     trainer_eval_loss: number | null;
-    /** The bar this attempt had to clear, frozen with its report. */
+    /**
+     * The pipeline's margin frozen with this attempt's report: the most the
+     * mean-change bar asks. The bar it actually had to clear is
+     * {@link LoopMetricPoint.mean_change_needed}.
+     */
     promote_margin: number | null;
     promote_min_win_rate: number | null;
+    /**
+     * The average score change this attempt had to clear, frozen with its
+     * report: `promote_margin`, lowered toward a quarter of the room left above
+     * what it was compared with near the top, never below what the comparison
+     * can tell from noise ({@link EvaluationMargin.mean_change_needed}). On a
+     * report written before that bar existed, or one that compared nothing, it
+     * is `promote_margin`. Null when the attempt has no report.
+     */
+    mean_change_needed: number | null;
     checkpoint_id: string | null;
     checkpoint_step: number | null;
     attempt_no: number | null;
@@ -2989,10 +3143,28 @@ export interface LoopSignalDay {
     total: number;
     by_verdict: Record<string, number>;
 }
+/**
+ * One day of a pipeline's samples, counted by their feedback today: `good`,
+ * `bad`, `corrected`, and `unreviewed` (no feedback from a person).
+ */
+export interface LoopSampleDay {
+    day: string;
+    good: number;
+    bad: number;
+    corrected: number;
+    unreviewed: number;
+}
 export interface LoopMetricsHistory {
     /** Oldest first, so a chart reads left to right. */
     runs: LoopMetricPoint[];
     signals: LoopSignalDay[];
+    /**
+     * The samples recorded each day over `days`, oldest first, by their
+     * feedback today: the pipeline's (the samples its Samples list holds) with
+     * `rule_id`, the workspace's without. Absent from a service from before it
+     * was sent.
+     */
+    samples?: LoopSampleDay[];
     /** True when older attempts exist beyond `limit`. */
     truncated: boolean;
 }
@@ -3247,6 +3419,11 @@ export interface TrainingRunCancelRequest {
 export interface TrainingRunListParams {
     rule_id?: string;
     state?: TrainingRunState;
+    /**
+     * Only the attempts that ended this way, in the words and values of
+     * `listPipelineAttempts`' `result`. Only numbered attempts have one.
+     */
+    result?: PipelineAttemptResultFilter;
     limit?: number;
     offset?: number;
 }
@@ -3313,9 +3490,22 @@ export interface TrainingRun {
      * True when a comparison machine was charged: the figure is then the most
      * it can have cost ("up to"), since it counts each machine from its first
      * boot where the platform bills from its first answer (and at the hourly
-     * cap when no placement price was reported).
+     * cap when no placement price was reported). False again once billing has
+     * settled every such machine and its charge for the time each one served
+     * replaced the "up to" figure (the run's timeline says what each was
+     * charged). The new model's machine of a run that went live (a version) or
+     * that a person took over is never settled, so it stays true for that run.
      */
     billed_candidate_up_to: boolean;
+    /**
+     * What was taken off for comparison machines that never started (never
+     * reached running): the platform charged nothing for them, so neither does
+     * the attempt -- say "Comparison machine: not charged (it never started)",
+     * or, while a comparison machine that ran is still charged beside it, that
+     * the one that never started was not charged. 0 when none; the cost figures
+     * beside it are already net of it.
+     */
+    comparison_returned_cents: number;
     spent_eval_cents: number;
     train_rows: number | null;
     holdout_rows: number | null;
@@ -3358,6 +3548,12 @@ export interface TrainingRun {
      * has ended.
      */
     training_summary: string | null;
+    /**
+     * What it trained on, as it was when its set was built: see
+     * {@link TrainedOn}. Null when its set is gone; absent from a service from
+     * before it was sent.
+     */
+    trained_on?: TrainedOn | null;
     candidate_deployment_id: string | null;
     candidate_name: string | null;
     candidate_seq: number;
@@ -3388,6 +3584,12 @@ export interface TrainingRun {
      * anything totalling what a run cost has to add this one too.
      */
     benchmark_spent_cents: number;
+    /**
+     * What scoring the samples nobody reviewed cost, in the passes the attempt
+     * took when it was booked. The fifth money column: the parts add up to the
+     * attempt's `cost_cents` on the pipeline only with it.
+     */
+    sample_scoring_cents?: number;
     alias_name: string | null;
     alias_written_at: string | null;
     serving_before: TrainingRuleServing | null;
@@ -3437,9 +3639,13 @@ export interface TrainingRunSummary {
     billed_candidate_cents: number;
     /** See {@link TrainingRun.billed_candidate_up_to}. */
     billed_candidate_up_to: boolean;
+    /** See {@link TrainingRun.comparison_returned_cents}. */
+    comparison_returned_cents: number;
     spent_eval_cents: number;
     /** The fourth money column. A row that leaves it out adds up short. */
     benchmark_spent_cents: number;
+    /** The fifth: scoring the samples nobody reviewed, taken when the attempt was booked. */
+    sample_scoring_cents?: number;
     training_job_id: string | null;
     candidate_deployment_id: string | null;
     evaluation_id: string | null;
@@ -3447,6 +3653,13 @@ export interface TrainingRunSummary {
     auto_promote_at: string | null;
     created_at: string;
     finished_at: string | null;
+    /**
+     * This run as its pipeline's attempts list carries it (outcome, cost, pass
+     * rates and the plain `explanation`), so the list of every pipeline's
+     * attempts says what each pipeline's own list says. Null for a run that is
+     * not a numbered attempt; absent from a service from before it was sent.
+     */
+    attempt?: PipelineAttempt | null;
 }
 /** One line of the timeline. detail holds codes, ids and cents, never prompt text. */
 export interface TrainingRunEvent {
@@ -3505,6 +3718,14 @@ export interface EvaluationMargin {
     promote_margin: number;
     promote_min_win_rate: number;
     min_holdout_rows: number;
+    /**
+     * The average score change this comparison actually asked of the new model:
+     * `promote_margin`, lowered toward a quarter of the room left above what it
+     * was compared with once that is smaller, never below what the
+     * comparison's size can tell from noise. Null when nothing was compared,
+     * and on a report written before, whose bar was `promote_margin`.
+     */
+    mean_change_needed: number | null;
 }
 /** The comparison report: the candidate against what serves today, same rows. */
 export interface Evaluation {
@@ -3737,7 +3958,7 @@ export type BenchmarkRunStatus = 'open' | 'done' | 'failed' | 'budget_stopped' |
  * is a missing number, never a lower one.
  */
 export interface BenchmarkScoring {
-    /** `judge_overall` (a scoring model) or `exact_match` (the exact check). */
+    /** `judge_overall` (the judge) or `exact_match` (the exact check). */
     metric: string;
     scale: string;
     aggregate: string;
@@ -3746,7 +3967,7 @@ export interface BenchmarkScoring {
 /**
  * How a benchmark's answers are scored, chosen when it is made and frozen:
  *
- * - `judge` (the default): a scoring model reads both answers against the
+ * - `judge` (the default): the judge reads both answers against the
  *   right answer, as for the held-back set.
  * - `exact`: the answer or the tool call must match the right answer, and no
  *   model is asked. When the right answer calls tools, the answer must make
@@ -3777,15 +3998,30 @@ export interface BenchmarkDimensionScore {
 /**
  * What a pipeline's history, or a benchmark's pinned set, takes up in storage
  * right now. Conversations, answers and scoring, and the training rows of a
- * pipeline's curated sets, are kept in storage only, never in the database. Not billed; it counts toward the workspace's storage
- * quota and is deleted with the pipeline (a benchmark's with the workspace).
- * `where` is null while nothing is stored.
+ * pipeline's curated sets, are kept in storage only, never in the database.
+ * Only the part of a pipeline's figure kept in the platform's storage is ever
+ * billed, as Conscious Loop storage at the storage rate with no daily minimum
+ * (each day's cost is added up and charged once it reaches a whole cent), and
+ * only where `billed` says so; deleting the pipeline stops it. The part in the
+ * workspace's own storage and the index in the database never are, and
+ * neither is a benchmark's pinned set, which is kept until the workspace is
+ * deleted (`billed` is always false on a benchmark). It counts toward the
+ * workspace's storage quota and is deleted with the pipeline (a benchmark's
+ * with the workspace). `where` is null while nothing is stored.
  */
 export type LoopStorageWhere = 'platform' | 'your_storage' | 'both';
 export interface LoopStorage {
     bytes: number;
     objects: number;
     where: LoopStorageWhere | null;
+    /**
+     * True when part of a pipeline's figure is in the platform's storage and
+     * this environment charges for that; false while nothing of it is charged
+     * (all of it in the workspace's own storage, nothing stored, an environment
+     * that only measures it for now, or a benchmark's pinned set, which is never
+     * billed).
+     */
+    billed: boolean;
 }
 /**
  * The frozen definition: the oracle, the scoring rule and the fingerprint of
@@ -4180,8 +4416,11 @@ export interface TrainingRuleMonthlyLimitRefusal {
     /**
      * The first instant of the next UTC month, when the month's count starts
      * again from nothing. When one attempt alone can cost more than the whole
-     * limit, no month will ever fit: saving the pipeline sets the limit again
-     * from its model's prices, and nothing else changes it; `message` says which.
+     * limit, no month will ever fit, and this is not a day the pipeline trains
+     * again: saving the pipeline sets the limit again from its model's prices,
+     * or -- when its next attempt is planned to need more training time than
+     * the whole limit allows -- a smaller model does, and a save plans the
+     * attempt again; `message` says which.
      */
     resumes_at: string;
 }
@@ -4220,10 +4459,13 @@ export interface PipelineActiveRun {
     /** What it has been billed so far, in cents. */
     cost_cents: number;
     /**
-     * True when a comparison machine was charged in `cost_cents`: the figure is
-     * the most it can have cost, so show it as "up to".
+     * True when a comparison machine was charged in `cost_cents` and billing
+     * has not settled it yet: the figure is the most it can have cost, so show
+     * it as "up to".
      */
     cost_up_to: boolean;
+    /** See {@link TrainingRun.comparison_returned_cents}. */
+    comparison_returned_cents: number;
     /** The version number it will take if it is put live. */
     will_be_version: number;
     /** Always null: nothing in flight is a version yet. */
@@ -4240,6 +4482,14 @@ export interface PipelineActiveRun {
      * not a version, and becomes one only if a member promotes it.
      */
     competed: boolean;
+    /** See {@link PipelineAttempt.sample_scoring_cents}. */
+    sample_scoring_cents?: number;
+    /** See {@link PipelineAttempt.sample_scored}. */
+    sample_scored?: number;
+    /** See {@link PipelineAttempt.sample_picked}. */
+    sample_picked?: number;
+    /** See {@link PipelineAttempt.train_rows_by_judge}. */
+    train_rows_by_judge?: number | null;
 }
 /**
  * One version: an attempt that was put live -- promoted by the pipeline's
@@ -4305,12 +4555,22 @@ export interface PipelineVersion {
      * True when `cost_cents` is the most the version can have cost rather than
      * what it did: its comparison machines at their ceilings
      * (`cost_includes_candidate_ceiling`), or one charged (counted from its
-     * first boot, where the platform bills from its first answer). Show it as
-     * "up to".
+     * first boot, where the platform bills from its first answer). A version's
+     * own new-model machine is the one that went live, and billing's settled
+     * figure never replaces its charge, so a version whose machine was charged
+     * stays "up to". Show it as "up to".
      */
     cost_up_to: boolean;
     /** Machine time the attempt held (training and comparison); null when unknown. */
     gpu_seconds: number | null;
+    /** See {@link PipelineAttempt.sample_scoring_cents}. */
+    sample_scoring_cents?: number;
+    /** See {@link PipelineAttempt.sample_scored}. */
+    sample_scored?: number;
+    /** See {@link PipelineAttempt.sample_picked}. */
+    sample_picked?: number;
+    /** See {@link PipelineAttempt.train_rows_by_judge}. */
+    train_rows_by_judge?: number | null;
     created_at: string;
     /**
      * When this version's head-to-head started. The champion it faced is the
@@ -4341,8 +4601,12 @@ export interface PipelineVersion {
  * learned from (`rebuild`).
  */
 export type PipelineAttemptTrigger = 'data' | 'schedule' | 'manual' | 'challenger' | 'rebuild';
-/** What became of an attempt. */
-export type PipelineAttemptOutcome = 'became_version' | 'not_better' | 'inconclusive' | 'stopped' | 'failed' | 'running' | 'waiting_for_review';
+/**
+ * What became of an attempt. `inconclusive` is a held-back test that ran and
+ * could not decide; `not_tested` is an attempt that ended before it was tested
+ * (its comparison machine never started, or it expired first).
+ */
+export type PipelineAttemptOutcome = 'became_version' | 'not_better' | 'inconclusive' | 'not_tested' | 'stopped' | 'failed' | 'running' | 'waiting_for_review';
 /** One attempt: every training run a pipeline makes, winner or not. */
 export interface PipelineAttempt {
     attempt: number;
@@ -4371,6 +4635,8 @@ export interface PipelineAttempt {
     cost_cents: number;
     /** See {@link PipelineVersion.cost_up_to}. */
     cost_up_to: boolean;
+    /** See {@link TrainingRun.comparison_returned_cents}. */
+    comparison_returned_cents: number;
     verdict: TrainingVerdict | null;
     win_rate: number | null;
     /**
@@ -4387,6 +4653,26 @@ export interface PipelineAttempt {
      * verdict and error. See {@link AttemptExplanation}.
      */
     explanation: AttemptExplanation | null;
+    /**
+     * What the judge's scoring of unreviewed samples cost before this attempt
+     * was made, in cents: already inside `cost_cents`, never on top of it. 0
+     * when nothing was scored for it.
+     */
+    sample_scoring_cents?: number;
+    /** How many unreviewed samples the judge scored for it. */
+    sample_scored?: number;
+    /** How many of those it picked (scored 0.8 or more). */
+    sample_picked?: number;
+    /**
+     * How many of `train_rows` were samples the judge picked rather than ones a
+     * person reviewed. Null on an attempt from before the judge picked samples.
+     */
+    train_rows_by_judge?: number | null;
+    /**
+     * What it trained on, as it was when its set was built: see
+     * {@link TrainedOn}. Null when its set is gone.
+     */
+    trained_on?: TrainedOn | null;
 }
 /**
  * An attempt's outcome in plain words: `headline` ("Could not be tested"),
@@ -4509,9 +4795,13 @@ export interface PipelineHoldout {
  * rows it trains on plus the ones held back to judge it. `new_rows`: `needed`
  * usable samples the last trained attempt did not have, which is
  * `floors.new_samples` (1,000), however many rows that attempt trained on.
- * `have`, `short` and `message` are
- * measured on the detail only. There, after an attempt has trained, `short`
- * is how many more new samples have to arrive: new samples fill the held-back
+ * On the detail, `have`, `short` and `message` are measured: the samples
+ * left once duplicates and conversations too long to train on are set aside.
+ * On each pipeline of the list they are counted, not measured: `have` is the
+ * count, and once it reaches `needed`, `short` is null (only the detail can
+ * say how many of them can be trained on) and `message` says so. Get the
+ * pipeline for the measured numbers. On the detail, after an attempt has
+ * trained, `short` is how many more new samples have to arrive: new samples fill the held-back
  * set first, so after the held-back share is raised it can be more than
  * `needed` - `have`; and `have` is the new samples the next attempt would
  * train on, or the samples since that attempt when that count is what it
@@ -4524,6 +4814,44 @@ export interface PipelineMinimums {
     have: number | null;
     short: number | null;
     message: string | null;
+    /**
+     * How many of `have` are samples the judge picked rather than ones a person
+     * reviewed (0 when none); `message` then says so in brackets.
+     */
+    picked_by_judge?: number;
+}
+/**
+ * Where the pipeline's automatic scoring of unreviewed samples is:
+ * - `scoring`: the judge is scoring samples nobody reviewed, newest first;
+ * - `waiting`: the next attempt has what it needs, or the pipeline is paused
+ *   or can make no more versions;
+ * - `idle`: nothing is left to score;
+ * - `held`: the platform held it for now (`held_until` says until when, when
+ *   it knows);
+ * - `off`: switched off on this pipeline.
+ */
+export type PipelineAutoScoringStatus = 'scoring' | 'waiting' | 'idle' | 'held' | 'off';
+/**
+ * The judge's scoring of the samples nobody reviewed. When the pipeline does
+ * not yet have the samples its next attempt needs, the judge scores
+ * unreviewed samples, newest first, with the rubric it tests attempts with;
+ * one scoring 0.8 or more (`threshold`) is trained on like a sample marked
+ * good, one below is not. A person's feedback always decides over it. What it
+ * cost shows on the attempt it was for (`sample_scoring_cents`).
+ *
+ * `picked`, `below_bar` and `unscored` count the unreviewed samples of the
+ * pipeline and add up to them. `message` is the service's own sentence: show
+ * it as written.
+ */
+export interface PipelineAutoScoring {
+    enabled: boolean;
+    status: PipelineAutoScoringStatus;
+    threshold: number;
+    picked: number;
+    below_bar: number;
+    unscored: number;
+    held_until: string | null;
+    message: string;
 }
 /**
  * The state of the live version's deployment:
@@ -4540,6 +4868,12 @@ export type PipelineLiveDeploymentStatus = 'serving' | 'starting' | 'stopped' |
 export interface PipelineLiveDeployment {
     status: PipelineLiveDeploymentStatus;
     checked_at: string;
+    /**
+     * The deployment the live version answers from, the one this check is
+     * about: its id, to open it on Deployments. Absent from a service from
+     * before it was sent.
+     */
+    deployment_id?: string;
 }
 /**
  * A rebuild on its way: a member deleted `conversations` conversations the
@@ -4586,6 +4920,31 @@ export interface Pipeline {
     own_tag: string | null;
     /** A conversation carrying ANY of these tags is this pipeline's data. */
     tags: string[];
+    /**
+     * The saved datasets it also trains on: a sample in any of them is its data
+     * too, as a sample carrying one of its tags is. Absent from a service from
+     * before datasets.
+     */
+    datasets?: PipelineDatasetRef[];
+    /**
+     * What narrows everything it trains on (its tags and its datasets alike).
+     * Every field empty or null: nothing narrows it. See
+     * {@link PipelineConditions}.
+     */
+    conditions?: PipelineConditions;
+    /**
+     * How many samples it has recorded, reviewed or not: its Samples list's
+     * total (`listTraces({ pipeline })`). Not what it trains on: a bad sample,
+     * one nobody reviewed yet and a held-back one are counted here too.
+     */
+    samples_total?: number;
+    /** How many of `samples_total` were recorded in the last 48 hours. */
+    samples_recent?: number;
+    /**
+     * The judge's scoring of its unreviewed samples. Null where the platform
+     * does not score samples. See {@link PipelineAutoScoring}.
+     */
+    auto_scoring?: PipelineAutoScoring | null;
     /**
      * The calls sent with `X-Loop-Pipeline` in the last `days` days that were
      * not recorded: under this pipeline's tags (an answer too large to keep, or
@@ -4733,6 +5092,19 @@ export interface TrainingRunResponse {
     verdict_checks_withheld: VerdictChecksWithheld | null;
     /** What the attempt has cost, part by part, from the charges recorded on it. */
     cost: TrainingRunCost;
+    /**
+     * What became of the attempt in plain words: the same object its row on
+     * the pipeline's attempts carries ({@link PipelineAttempt.explanation}), so
+     * an attempt older than the latest a pipeline carries still says it. Null
+     * for a run that is not a numbered attempt; absent from a service from
+     * before it was sent.
+     */
+    explanation?: AttemptExplanation | null;
+    /**
+     * True when this service lists an attempt's own samples
+     * (`listTraces({ attempt, split })`). Absent from a service that cannot.
+     */
+    traces_filter_attempt?: boolean;
 }
 /** Why a run detail's `verdict_checks` is withheld. */
 export type VerdictChecksWithheld = 'decided_under_earlier_rule';
@@ -4785,8 +5157,22 @@ export interface TrainingRunCost {
     comparison_cents: number;
     scoring_cents: number;
     benchmark_cents: number;
+    /**
+     * What the judge's scoring of unreviewed samples cost before the attempt
+     * was made ({@link PipelineAttempt.sample_scoring_cents}): a part of
+     * `total_cents` like the others. Absent from a service from before it was
+     * sent.
+     */
+    sample_scoring_cents?: number;
     total_cents: number;
     up_to: boolean;
+    /**
+     * What was taken off `comparison_cents` for comparison machines that never
+     * started: "Comparison machine: not charged (it never started)" (with
+     * `comparison_cents` above 0, only the one that never started). 0 when
+     * none.
+     */
+    comparison_returned_cents: number;
 }
 export interface TrainingRunActionResponse {
     run: TrainingRun;
@@ -4857,7 +5243,7 @@ export interface PipelineResponse {
     pipeline: Pipeline;
 }
 /** What became of an attempt, as `listPipelineAttempts` filters it. */
-export type PipelineAttemptResultFilter = 'became_version' | 'not_better' | 'inconclusive' | 'failed' | 'stopped' | 'running' | 'waiting_for_review';
+export type PipelineAttemptResultFilter = 'became_version' | 'not_better' | 'inconclusive' | 'not_tested' | 'failed' | 'stopped' | 'running' | 'waiting_for_review';
 /** Which model's attempts: the pipeline's own (`main`) or its second model's (`second`). */
 export type PipelineAttemptModelFilter = 'main' | 'second';
 export interface PipelineAttemptListParams {
@@ -4894,6 +5280,30 @@ export interface PipelineWriteRequest {
     } | null;
     /** More tags whose conversations the pipeline trains on. On an edit the list replaces the old one. */
     tags?: string[];
+    /**
+     * The saved datasets (`listDatasets`) it also trains on, by id, at most 32.
+     * Absent leaves them; `[]` takes every one off; a list replaces them. A
+     * change applies from the next attempt: nothing is trained again, and
+     * versions already made keep what they learned.
+     */
+    datasets?: string[];
+    /**
+     * What narrows everything it trains on. Absent leaves them; null clears
+     * them; an object replaces them. `feedback`, when set, must include `good`
+     * or `corrected` (refused 400 otherwise). A sample the judge picked has no
+     * feedback from a person, so it counts as `unreviewed` here: a list without
+     * `unreviewed` trains only on samples a person reviewed. A change applies
+     * from the next attempt, as for `datasets`.
+     */
+    conditions?: PipelineConditionsWrite | null;
+    /**
+     * The judge's scoring of unreviewed samples: on by default.
+     * `{ enabled: false }` trains only on samples a person reviewed. No other
+     * key is taken. It is not a training change: nothing is asked again.
+     */
+    auto_scoring?: {
+        enabled: boolean;
+    };
     /**
      * The schedule to wait for, beside the platform's fixed sample minimum.
      * There is no `min_samples`: the minimums are fixed, and a body that sends
@@ -5022,7 +5432,7 @@ export interface RuleBenchmark {
     benchmark_status: string;
     item_count: number;
     items_digest: string;
-    /** How it scores an answer: a scoring model reading its rubric (`judge`) or an exact check against its right answer (`exact`). */
+    /** How it scores an answer: the judge reading its rubric (`judge`) or an exact check against its right answer (`exact`). */
     check: BenchmarkCheck;
     /** What its pinned set takes up in storage now: the figure the benchmark itself carries, this workspace's alone. */
     storage: LoopStorage;
@@ -5177,7 +5587,7 @@ export type AttemptDirection = 'higher' | 'lower' | 'same';
 export interface AttemptHoldout {
     evaluation_id: string | null;
     status: EvaluationStatus | null;
-    /** The scoring model its comparison froze when it opened; null for an attempt never compared. */
+    /** The judge its comparison froze when it opened; null for an attempt never compared. */
     judge_model: string | null;
     rows_scored: number | null;
     wins: number | null;
@@ -5213,7 +5623,7 @@ export interface AttemptCompareSide {
  * The held-back conversations both attempts were scored on (`rows`), matched
  * by the conversation they came from.
  *
- * `comparable` says both were scored the same way: the same scoring model and
+ * `comparable` says both were scored the same way: the same judge and
  * the same rubric, both recorded. When not, `why` says so, naming both scoring
  * models, and `higher`, `lower`, `same`, `a` and `b` are null: the two scores
  * are not on one scale. `answer_changed` counts the conversations whose right
@@ -5332,3 +5742,191 @@ export type PipelineImportParams = Omit<LoopImportParams, 'source'> & {
     source?: string;
     default_feedback?: 'good';
 };
+/**
+ * The feedback words a dataset or a pipeline's conditions select by, read as
+ * the Samples list reads them: the deciding feedback from a person. A sample
+ * no person gave feedback on is `unreviewed`, whether or not the judge scored
+ * it.
+ */
+export type LoopDatasetFeedback = 'good' | 'corrected' | 'bad' | 'unreviewed';
+/** Where a sample came from: your app (`captured`) or an imported file (`imported`). */
+export type LoopSampleOrigin = 'captured' | 'imported';
+/**
+ * Samples counted by the feedback a person gave them, in the Samples list's
+ * words: a dataset's or a preview's (they add up to its `samples`) and the
+ * workspace's on `stats` (they add up to `traces`).
+ */
+export interface LoopFeedbackCounts {
+    good: number;
+    corrected: number;
+    bad: number;
+    unreviewed: number;
+}
+/** A pipeline, named: one that trains on a dataset, or that a dataset was taken off. */
+export interface LoopDatasetPipelineRef {
+    rule_id: string;
+    name: string;
+}
+/**
+ * A saved dataset (`GET /api/loop/datasets/{id}`). Every condition is
+ * optional and they all hold together; a dataset with none is every sample in
+ * the workspace.
+ */
+export interface LoopDataset {
+    id: string;
+    name: string;
+    /** The tags it selects by; a tag selects its child tags too. May be empty. */
+    tags: string[];
+    /** `any`: a sample carrying at least one of `tags`; `all`: every one. `any` when `tags` is empty. */
+    match: 'any' | 'all';
+    /** The feedback words it keeps; empty keeps every sample. */
+    feedback: LoopDatasetFeedback[];
+    origin: LoopSampleOrigin | null;
+    /** The model that answered, exactly. */
+    model: string | null;
+    /** When a sample was recorded, RFC 3339, both ends included. */
+    from: string | null;
+    to: string | null;
+    /** Moves on every edit; send it back as `expected_revision`. */
+    revision: number;
+    /** How many samples it selects now: the same number `listTraces({ dataset })` totals. */
+    samples: number;
+    feedback_counts: LoopFeedbackCounts;
+    /** How many of `samples` are in a benchmark, and so are never trained on. */
+    in_benchmarks: number;
+    /** The pipelines that train on it. */
+    used_by: LoopDatasetPipelineRef[];
+    created_at: string;
+    updated_at: string;
+}
+/**
+ * The body of `updateDataset`: every field optional. Unknown keys are refused
+ * 400 naming them. An absent key is left as it is, null clears `origin`,
+ * `model`, `from` and `to`, and `[]` clears `tags` and `feedback`.
+ * `createDataset` takes {@link LoopDatasetCreate}.
+ */
+export interface LoopDatasetWrite {
+    /** 1 to 80 characters, unique in the workspace whatever its case (409 `DATASET_NAME_TAKEN`). */
+    name?: string;
+    /** At most 32. */
+    tags?: string[];
+    match?: 'any' | 'all';
+    feedback?: LoopDatasetFeedback[];
+    origin?: LoopSampleOrigin | null;
+    /** 1 to 200 characters. */
+    model?: string | null;
+    /** RFC 3339; `from` may not be after `to`. */
+    from?: string | null;
+    to?: string | null;
+    /** An edit's compare-and-set: refused 409 `REVISION_MISMATCH` when the dataset moved on. */
+    expected_revision?: number;
+}
+/**
+ * The body of `createDataset`: a `name`, which the service always needs, and
+ * any of the conditions. There is no revision to compare against yet.
+ */
+export type LoopDatasetCreate = Omit<LoopDatasetWrite, 'name' | 'expected_revision'> & {
+    name: string;
+};
+/**
+ * What `previewDataset` counts: a dataset's conditions without its name, and
+ * `datasets`, saved datasets whose samples are added (as a pipeline adds
+ * them), so a pipeline's tags, datasets and conditions can be counted in one
+ * call before they are saved.
+ */
+export interface LoopDatasetPreviewParams {
+    tags?: string[];
+    match?: 'any' | 'all';
+    feedback?: LoopDatasetFeedback[];
+    origin?: LoopSampleOrigin | null;
+    model?: string | null;
+    from?: string | null;
+    to?: string | null;
+    datasets?: string[];
+}
+/** One model that answered samples of a preview, and how many. */
+export interface LoopDatasetPreviewModel {
+    model: string;
+    samples: number;
+}
+/**
+ * The counts a preview answers. They are before duplicates and conversations
+ * too long to train on are set aside, which only an attempt does: a
+ * pipeline's own `minimums` are the exact training numbers.
+ */
+export interface LoopDatasetPreview {
+    samples: number;
+    feedback_counts: LoopFeedbackCounts;
+    in_benchmarks: number;
+    /** The 20 models that answered most of `samples`, most first. */
+    models: LoopDatasetPreviewModel[];
+}
+export interface LoopDatasetListParams {
+    /** 50 when absent; at most 200. */
+    limit?: number;
+    offset?: number;
+}
+/** A page of the workspace's datasets, the most recently changed first. */
+export interface LoopDatasetListResponse {
+    datasets: LoopDataset[];
+    total: number;
+    limit: number;
+    offset: number;
+}
+/**
+ * `deleteDataset`: the samples stay; `removed_from` names every pipeline
+ * that stops training on its samples from its next attempt.
+ */
+export interface LoopDatasetDeleteResponse {
+    deleted: boolean;
+    removed_from: LoopDatasetPipelineRef[];
+}
+/** A dataset a pipeline trains on, by id and name. */
+export interface PipelineDatasetRef {
+    id: string;
+    name: string;
+}
+/**
+ * What narrows everything a pipeline trains on: the feedback words it keeps
+ * (empty keeps every one), where samples came from, the model that answered,
+ * and when they were recorded. A pipeline's tags are its `tags` field, never a
+ * condition.
+ */
+export interface PipelineConditions {
+    feedback: LoopDatasetFeedback[];
+    origin: LoopSampleOrigin | null;
+    model: string | null;
+    from: string | null;
+    to: string | null;
+}
+/** The conditions a pipeline create or edit sends: an object replaces them whole. */
+export interface PipelineConditionsWrite {
+    feedback?: LoopDatasetFeedback[];
+    origin?: LoopSampleOrigin | null;
+    model?: string | null;
+    from?: string | null;
+    to?: string | null;
+}
+/** A dataset as an attempt trained on it: its conditions at the revision it had then. */
+export interface TrainedOnDataset {
+    id: string;
+    name: string;
+    revision: number;
+    tags: string[];
+    match: 'any' | 'all';
+    feedback: LoopDatasetFeedback[];
+    origin: LoopSampleOrigin | null;
+    model: string | null;
+    from: string | null;
+    to: string | null;
+}
+/**
+ * What an attempt trained on, read from its own set as it was built, never
+ * from today's settings: the pipeline's tags, its conditions, and each
+ * dataset as it was then. A later edit of a dataset never changes it.
+ */
+export interface TrainedOn {
+    tags: string[];
+    conditions: PipelineConditions;
+    datasets: TrainedOnDataset[];
+}