runbios-sdk 0.2.1-dev.152 → 0.2.1-dev.158

package/dist/index.d.ts

@@ -36,7 +36,7 @@ import { Loop } from './resources/loop.js';
  * SDK version. Sent as part of the User-Agent header.
  * Must match package.json "version" -- enforced by a contract test.
  */
-export declare const VERSION = "0.2.1-dev.152";
+export declare const VERSION = "0.2.1-dev.158";
 export declare class RunBiOS {
     /** Search models, fetch configs, check adapter compatibility. */
     readonly models: Models;
@@ -75,6 +75,6 @@ export { Training } from './resources/training.js';
 export { Wallet } from './resources/wallet.js';
 export { GPU, type GPURecommendation } from './resources/gpu.js';
 export { Inference, validateChatRequest, parseSSE, buildInferenceRequest, contextSizingBasis, type ContextSizingBasis, CONTEXT_DEFAULT_CEILING, CONTEXT_EDITABLE_FLOOR, type ChatMessage, type FunctionTool, type ChatCompletionParams, type ChatCompletionResponse, type ChatCompletionChunk, } from './resources/inference.js';
-export type { BiOSConfig, PaginatedResponse, ApiErrorBody, AvailableGpuAlternative, CapacityMinimumRequirement, InferenceBookingAccepted, Model, ModelSearchParams, ModelSearchResponse, ModelDetailResponse, ModelConfig, Dataset, DatasetListParams, DatasetListResponse, DatasetUploadParams, DatasetPreview, DatasetPreviewParams, DatasetImportHFParams, DatasetRegisterHFParams, DatasetHubSearchParams, DatasetHubPreviewParams, DatasetValidation, DatasetFormatVariant, DatasetFormatSpec, DatasetFormatSpecs, DatasetStorageUsage, TrainingMethod, RLHFAlgorithm, AdapterType, TrainingJobStatus, TrainingStatusFilter, TrainingCreateParams, TrainingListParams, TrainingListResponse, TrainingJob, TrainingMetrics, TrainingResourceMetrics, TrainingDeviceMetrics, MetricPoint, MetricGraphConfig, TrainingCheckpoint, TrainingLogs, TrainingLogEntry, TrainingStopResponse, TrainingResumeResponse, CanonicalTrainingRequest, TrainingPreflightDataset, TrainingPreflightWarning, TrainingPreflightResponse, TrainingCapabilityChoice, TrainingConfigFieldCapability, TrainingCapabilities, GPUChoice, WalletBalance, Transaction, TransactionListResponse, TransactionListParams, GPUInfo, GPUPricingResponse, GPUOptionsParams, TrainingRecommendParams, TrainingAdvisorVerdict, TrainingAdvisorRecommendation, TrainingAdvisorUserShape, TrainingAdvisorJustification, TrainingAdvisorBasis, GPUOption, GPUOptionSuggestion, GPUOptionsResponse, InferenceStatus, InferenceCreateParams, InferenceUpdateParams, InferenceDeployment, InferenceDeploymentSummary, InferenceCreateResponse, InferenceListResponse, InferenceUpdateResponse, InferenceLifecycleResponse, InferenceDeleteResponse, InferenceGPUOptionMarket, InferenceGPUOption, InferenceGPUAlternative, InferenceGPUOptionsParams, InferenceGPUOptionsResponse, InferenceCanonicalRequest, InferencePreflightWarning, InferencePreflightResponse, AdapterCompatibility, AdapterCompatibilityResponse, AdapterCompatibilityParams, SupportedArchitecture, ArchitectureScope, SupportedArchitecturesResponse, ApiKey, ApiKeyScope, ApiKeyIntrospection, Organization, OrgMember, OrgInvite, Workspace, Integration, IntegrationCreateParams, StorageUsage, StorageObject, PromptTokensDetails, ChatCompletionUsage, TrainingCadence, TrainingCombinator, ServingKind, TrainingRulePausedReason, LoopTrainingMethod, TrainType, GradersScope, TrainingTrigger, TrainingRunState, TrainingVerdict, TrainingDecision, NotifyState, EvaluationStatus, EvaluationItemStatus, EvaluationWinner, EvaluationWarningCode, ConsentVia, TrainingToolCall, TrainingMessage, TrainingRuleServing, TrainingGPURung, TrainingRule, TrainingRuleConsent, TrainingRulePreflightRefusal, TrainingRuleKeyRef, TrainingRuleKeyCheck, TrainingRulePreflight, TrainingRuleBuildSpec, TrainingRuleTriggerInput, TrainingRuleTrainingInput, TrainingRuleDeployInput, TrainingRuleMoneyInput, TrainingRuleEvaluationInput, TrainingRulePromotionInput, TrainingRuleAcceptTerms, TrainingRulePreflightRequest, TrainingRuleCreateRequest, TrainingRuleUpdateRequest, TrainingRuleConsentRequest, TrainingRuleListParams, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, TrainingRunListParams, TrainingRun, TrainingRunSummary, TrainingRunEvent, TrainingRunLinks, TrainingRunActions, EvaluationRef, EvaluationDecoding, EvaluationJudgeDimension, EvaluationDimensionScore, EvaluationGraderScore, EvaluationWarning, EvaluationMargin, JudgeAgreementDimension, JudgeAgreement, Evaluation, EvaluationItemSide, EvaluationItem, EvaluationItemListParams, JudgeAgreementParams, AgentSettings, AgentSettingsRequest, InferenceAlias, InferenceAliasRequest, TrainingRuleListResponse, TrainingRuleResponse, TrainingRuleMutationResponse, TrainingRuleDeleteResponse, TrainingRunListResponse, TrainingRunResponse, TrainingRunActionResponse, EvaluationItemsResponse, AgentSettingsResponse, InferenceAliasListResponse, InferenceAliasResponse, InferenceAliasDeleteResponse, } from './types.js';
+export type { BiOSConfig, PaginatedResponse, ApiErrorBody, AvailableGpuAlternative, CapacityMinimumRequirement, InferenceBookingAccepted, Model, ModelSearchParams, ModelSearchResponse, ModelDetailResponse, ModelConfig, Dataset, DatasetListParams, DatasetListResponse, DatasetUploadParams, DatasetPreview, DatasetPreviewParams, DatasetImportHFParams, DatasetRegisterHFParams, DatasetHubSearchParams, DatasetHubPreviewParams, DatasetValidation, DatasetFormatVariant, DatasetFormatSpec, DatasetFormatSpecs, DatasetStorageUsage, TrainingMethod, RLHFAlgorithm, AdapterType, TrainingJobStatus, TrainingStatusFilter, TrainingCreateParams, TrainingListParams, TrainingListResponse, TrainingJob, TrainingMetrics, TrainingResourceMetrics, TrainingDeviceMetrics, MetricPoint, MetricGraphConfig, TrainingCheckpoint, TrainingLogs, TrainingLogEntry, TrainingStopResponse, TrainingResumeResponse, CanonicalTrainingRequest, TrainingPreflightDataset, TrainingPreflightWarning, TrainingPreflightResponse, TrainingCapabilityChoice, TrainingConfigFieldCapability, TrainingCapabilities, GPUChoice, WalletBalance, Transaction, TransactionListResponse, TransactionListParams, GPUInfo, GPUPricingResponse, GPUOptionsParams, TrainingRecommendParams, TrainingAdvisorVerdict, TrainingAdvisorRecommendation, TrainingAdvisorUserShape, TrainingAdvisorJustification, TrainingAdvisorBasis, GPUOption, GPUOptionSuggestion, GPUOptionsResponse, InferenceStatus, InferenceCreateParams, InferenceUpdateParams, InferenceDeployment, InferenceDeploymentSummary, InferenceCreateResponse, InferenceListResponse, InferenceUpdateResponse, InferenceLifecycleResponse, InferenceDeleteResponse, InferenceGPUOptionMarket, InferenceGPUOption, InferenceGPUAlternative, InferenceGPUOptionsParams, InferenceGPUOptionsResponse, InferenceCanonicalRequest, InferencePreflightWarning, InferencePreflightResponse, AdapterCompatibility, AdapterCompatibilityResponse, AdapterCompatibilityParams, SupportedArchitecture, ArchitectureScope, SupportedArchitecturesResponse, ApiKey, ApiKeyScope, ApiKeyIntrospection, Organization, OrgMember, OrgInvite, Workspace, Integration, IntegrationCreateParams, StorageUsage, StorageObject, PromptTokensDetails, ChatCompletionUsage, TrainingCadence, TrainingCombinator, ServingKind, TrainingRulePausedReason, LoopTrainingMethod, TrainType, GradersScope, TrainingTrigger, TrainingRunState, TrainingVerdict, TrainingDecision, NotifyState, EvaluationStatus, EvaluationItemStatus, EvaluationWinner, EvaluationWarningCode, ConsentVia, TrainingToolCall, TrainingMessage, TrainingRuleServing, TrainingGPURung, TrainingRule, TrainingRuleConsent, TrainingRulePreflightRefusal, TrainingRuleKeyRef, TrainingRuleKeyCheck, TrainingRulePreflight, TrainingRuleBuildSpec, TrainingRuleTriggerInput, TrainingRuleTrainingInput, TrainingRuleDeployInput, TrainingRuleMoneyInput, TrainingRuleEvaluationInput, TrainingRulePromotionInput, TrainingRuleAcceptTerms, TrainingRulePreflightRequest, TrainingRuleCreateRequest, TrainingRuleUpdateRequest, TrainingRuleConsentRequest, TrainingRuleListParams, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, TrainingRunListParams, TrainingRun, TrainingRunSummary, TrainingRunEvent, TrainingRunLinks, TrainingRunActions, EvaluationRef, EvaluationDecoding, EvaluationJudgeDimension, EvaluationDimensionScore, EvaluationGraderScore, EvaluationWarning, EvaluationMargin, JudgeAgreementDimension, JudgeAgreement, Evaluation, EvaluationItemSide, EvaluationItem, EvaluationItemListParams, JudgeAgreementParams, AgentSettings, AgentSettingsRequest, BenchmarkStatus, BenchmarkSourceKind, BenchmarkRunStatus, BenchmarkScoring, BenchmarkDimensionScore, Benchmark, BenchmarkItem, BenchmarkRun, BenchmarkHistoryPoint, BenchmarkSource, BenchmarkCreateParams, BenchmarkListParams, BenchmarkItemListParams, BenchmarkHistoryParams, BenchmarkRetireParams, TrainingRuleBenchmarkRequest, InferenceAlias, InferenceAliasRequest, TrainingRuleListResponse, TrainingRuleResponse, TrainingRuleMutationResponse, TrainingRuleDeleteResponse, TrainingRunListResponse, TrainingRunResponse, TrainingRunActionResponse, EvaluationItemsResponse, AgentSettingsResponse, InferenceAliasListResponse, InferenceAliasResponse, InferenceAliasDeleteResponse, BenchmarkListResponse, BenchmarkItemsResponse, BenchmarkHistoryResponse, } from './types.js';
 /** The closed set of run states a training run never leaves. */
 export { TERMINAL_RUN_STATES } from './types.js';

package/dist/index.js

@@ -36,7 +36,7 @@ import { Loop } from './resources/loop.js';
  * SDK version. Sent as part of the User-Agent header.
  * Must match package.json "version" -- enforced by a contract test.
  */
-export const VERSION = '0.2.1-dev.152';
+export const VERSION = '0.2.1-dev.158';
 export class RunBiOS {
     /** Search models, fetch configs, check adapter compatibility. */
     models;

package/dist/resources/loop.d.ts

@@ -1,5 +1,5 @@
 import type { HttpClient } from '../client.js';
-import type { LoopTrace, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopImportParams, LoopImportResult, LoopSignal, LoopSignalParams, LoopCandidate, LoopCandidateParams, LoopGrader, LoopGraderParams, LoopLabel, LoopLabelCount, LoopGradeResult, LoopDataset, LoopDatasetListParams, LoopDatasetItem, LoopDatasetCreateParams, LoopConfig, LoopConfigParams, LoopBuildRule, LoopBuildRuleParams, LoopBuildRuleUpdateParams, LoopStats, LoopJudge, LoopJudgeParams, LoopAgentCredential, LoopAgentStatus, LoopSampleRun, LoopSampleRunParams, LoopJudgeRun, LoopJudgeRunItems, LoopJudgeRunItemStatus, LoopJudgeWork, LoopJudgeVerdict, LoopJudgeVerdictResult, TrainingRule, TrainingRulePreflight, TrainingRulePreflightRequest, TrainingRuleCreateRequest, TrainingRuleUpdateRequest, TrainingRuleConsentRequest, TrainingRuleListParams, TrainingRuleListResponse, TrainingRuleResponse, TrainingRuleMutationResponse, TrainingRuleDeleteResponse, TrainingRun, TrainingRunListParams, TrainingRunListResponse, TrainingRunResponse, TrainingRunActionResponse, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, Evaluation, EvaluationItemListParams, EvaluationItemsResponse, JudgeAgreement, JudgeAgreementParams, AgentSettings, AgentSettingsRequest } from '../types.js';
+import type { LoopTrace, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopImportParams, LoopImportResult, LoopSignal, LoopSignalParams, LoopCandidate, LoopCandidateParams, LoopGrader, LoopGraderParams, LoopLabel, LoopLabelCount, LoopGradeResult, LoopDataset, LoopDatasetListParams, LoopDatasetItem, LoopDatasetCreateParams, LoopConfig, LoopConfigParams, LoopBuildRule, LoopBuildRuleParams, LoopBuildRuleUpdateParams, LoopStats, LoopJudge, LoopJudgeParams, LoopAgentCredential, LoopAgentStatus, LoopSampleRun, LoopSampleRunParams, LoopJudgeRun, LoopJudgeRunItems, LoopJudgeRunItemStatus, LoopJudgeWork, LoopJudgeVerdict, LoopJudgeVerdictResult, TrainingRule, TrainingRulePreflight, TrainingRulePreflightRequest, TrainingRuleCreateRequest, TrainingRuleUpdateRequest, TrainingRuleConsentRequest, TrainingRuleListParams, TrainingRuleListResponse, TrainingRuleResponse, TrainingRuleMutationResponse, TrainingRuleDeleteResponse, TrainingRun, TrainingRunListParams, TrainingRunListResponse, TrainingRunResponse, TrainingRunActionResponse, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, Evaluation, EvaluationItemListParams, EvaluationItemsResponse, JudgeAgreement, JudgeAgreementParams, Benchmark, BenchmarkCreateParams, BenchmarkHistoryParams, BenchmarkHistoryResponse, BenchmarkItemListParams, BenchmarkItemsResponse, BenchmarkListParams, BenchmarkListResponse, BenchmarkRetireParams, BenchmarkRun, TrainingRuleBenchmarkRequest, AgentSettings, AgentSettingsRequest } from '../types.js';
 /**
  * The Conscious Loop -- capture what your model was asked and answered, record
  * whether it was right, and turn those judgements into training data.
@@ -740,6 +740,98 @@ export declare class Loop {
      * reviewer overlap yet" is an answer, and 100% of two pairs is not.
      */
     getJudgeAgreement(judgeId: string, params?: JudgeAgreementParams): Promise<JudgeAgreement>;
+    /**
+     * Pin a fixed set of conversations, with a judge frozen beside them, and
+     * measure every future run against it.
+     *
+     * The comparison answers "is this candidate better than what serves today".
+     * It cannot answer "is my model getting better", because its rows, its
+     * opponent and its judge all move between runs. A benchmark is the other
+     * instrument: the same conversations, the same oracle, the same decoding,
+     * replayed against both models on every run that finishes its comparison,
+     * and reported as two absolute numbers on a scale that does not move.
+     *
+     * 10 to 200 conversations. A named conversation with nothing to ask a model
+     * is refused rather than skipped, because pinning 47 of the 50 you chose is
+     * the set being wrong from the first day and you would never find out.
+     *
+     * It raises no amount you have already agreed to. The replay's calls come
+     * out of the rule's existing `eval_ceiling_cents`, and
+     * `per_run_ceiling_cents` can only lower what is spent inside that.
+     */
+    createBenchmark(params: BenchmarkCreateParams): Promise<Benchmark>;
+    /** Your benchmarks, newest first. Filter by `status` to hide retired ones. */
+    listBenchmarks(params?: BenchmarkListParams): Promise<BenchmarkListResponse>;
+    /**
+     * One benchmark: the frozen oracle, the scoring rule and the fingerprint of
+     * the set.
+     *
+     * `items_digest` is recomputed from the rows and compared at the start of
+     * every replay, so "the set cannot drift" is something you can check rather
+     * than something the platform promises.
+     */
+    getBenchmark(id: string): Promise<Benchmark>;
+    /**
+     * The pinned conversations, paged. `limit` is capped at 100 by the service.
+     *
+     * `source_trace_id` and `source_trace_url` come back null once the
+     * conversation a row was copied from has been deleted. The row itself stays
+     * and every number already measured against it stays exactly as comparable
+     * as it was: retention cannot shrink a benchmark.
+     */
+    listBenchmarkItems(id: string, params?: BenchmarkItemListParams): Promise<BenchmarkItemsResponse>;
+    /**
+     * The trend: every score this benchmark has produced, newest first, each
+     * beside the checkpoint that produced it and what you then did about it.
+     *
+     * EVERY REPLAY IS A POINT, including the ones with no numbers on them. A
+     * series that silently dropped the runs where the benchmark could not be
+     * scored would read as an unbroken line and not be one, so those points come
+     * back with `candidate_score` null and `status_reason` saying why.
+     */
+    getBenchmarkHistory(id: string, params?: BenchmarkHistoryParams): Promise<BenchmarkHistoryResponse>;
+    /**
+     * Stop replaying a benchmark, and detach it from every rule that names it.
+     *
+     * Nothing measured is removed. There is no delete and no update on purpose:
+     * the series of numbers measured against a set is what a benchmark is for,
+     * so an edit would make everything before it incomparable with everything
+     * after it and a delete would throw the series away. Changing the set means
+     * pinning a new benchmark, and retiring the old one releases its name.
+     *
+     * Retiring twice is somebody pressing a button twice: the second call
+     * answers with the retired benchmark rather than a refusal.
+     */
+    retireBenchmark(id: string, params?: BenchmarkRetireParams): Promise<Benchmark>;
+    /**
+     * One replay in full: both absolute scores, the difference between them on
+     * the same set, the per-dimension breakdown and what it cost.
+     *
+     * Read `status` before you read the scores. A replay that did not score
+     * every pinned conversation publishes no score at all -- all three of
+     * `candidate_score`, `incumbent_score` and `score_delta` are null and
+     * `status_reason` is the sentence that explains it -- because a mean over
+     * whichever conversations happened to succeed is a measurement of a
+     * different set, which is the exact defect a standing benchmark removes.
+     */
+    getBenchmarkRun(id: string): Promise<BenchmarkRun>;
+    /**
+     * Attach a benchmark to a rule, or send `{ benchmark_id: null }` to detach
+     * it.
+     *
+     * Its own route rather than a field on the rule body, because it is a
+     * decision to replay a fixed set on every future run of this rule for as
+     * long as it stands, and its refusals -- retired, or belonging to another
+     * workspace -- are about the benchmark rather than about the rule.
+     *
+     * It does not invalidate consent and the reply says so: attaching raises
+     * neither the amount set aside for judge calls nor the amount set aside for
+     * keeping the new model available, so nobody is asked to read the same
+     * sentence again. A retired benchmark is refused with `BENCHMARK_RETIRED`,
+     * because a rule pointed at one would report no number on every run and say
+     * nothing about why.
+     */
+    setTrainingRuleBenchmark(id: string, params: TrainingRuleBenchmarkRequest): Promise<TrainingRuleMutationResponse>;
     /**
      * The workspace's agent options: which model it defaults to, the system
      * prompts it judges and samples with, and the monthly cap on what its model

package/dist/resources/loop.js

@@ -1031,6 +1031,139 @@ export class Loop {
         const qs = q.toString();
         return this._http.fetchGet(`/api/loop/judges/${encodeURIComponent(judgeId)}/agreement${qs ? `?${qs}` : ''}`);
     }
+    // ── the standing benchmark ────────────────────────────────────────────
+    /**
+     * Pin a fixed set of conversations, with a judge frozen beside them, and
+     * measure every future run against it.
+     *
+     * The comparison answers "is this candidate better than what serves today".
+     * It cannot answer "is my model getting better", because its rows, its
+     * opponent and its judge all move between runs. A benchmark is the other
+     * instrument: the same conversations, the same oracle, the same decoding,
+     * replayed against both models on every run that finishes its comparison,
+     * and reported as two absolute numbers on a scale that does not move.
+     *
+     * 10 to 200 conversations. A named conversation with nothing to ask a model
+     * is refused rather than skipped, because pinning 47 of the 50 you chose is
+     * the set being wrong from the first day and you would never find out.
+     *
+     * It raises no amount you have already agreed to. The replay's calls come
+     * out of the rule's existing `eval_ceiling_cents`, and
+     * `per_run_ceiling_cents` can only lower what is spent inside that.
+     */
+    async createBenchmark(params) {
+        const res = await this._http.fetchPost('/api/loop/benchmarks', params);
+        return res.benchmark;
+    }
+    /** Your benchmarks, newest first. Filter by `status` to hide retired ones. */
+    async listBenchmarks(params = {}) {
+        const q = new URLSearchParams();
+        if (params.status)
+            q.set('status', params.status);
+        if (params.limit != null)
+            q.set('limit', String(params.limit));
+        if (params.offset != null)
+            q.set('offset', String(params.offset));
+        const qs = q.toString();
+        return this._http.fetchGet(`/api/loop/benchmarks${qs ? `?${qs}` : ''}`);
+    }
+    /**
+     * One benchmark: the frozen oracle, the scoring rule and the fingerprint of
+     * the set.
+     *
+     * `items_digest` is recomputed from the rows and compared at the start of
+     * every replay, so "the set cannot drift" is something you can check rather
+     * than something the platform promises.
+     */
+    async getBenchmark(id) {
+        const res = await this._http.fetchGet(`/api/loop/benchmarks/${encodeURIComponent(id)}`);
+        return res.benchmark;
+    }
+    /**
+     * The pinned conversations, paged. `limit` is capped at 100 by the service.
+     *
+     * `source_trace_id` and `source_trace_url` come back null once the
+     * conversation a row was copied from has been deleted. The row itself stays
+     * and every number already measured against it stays exactly as comparable
+     * as it was: retention cannot shrink a benchmark.
+     */
+    async listBenchmarkItems(id, params = {}) {
+        const q = new URLSearchParams();
+        if (params.limit != null)
+            q.set('limit', String(params.limit));
+        if (params.offset != null)
+            q.set('offset', String(params.offset));
+        const qs = q.toString();
+        return this._http.fetchGet(`/api/loop/benchmarks/${encodeURIComponent(id)}/items${qs ? `?${qs}` : ''}`);
+    }
+    /**
+     * The trend: every score this benchmark has produced, newest first, each
+     * beside the checkpoint that produced it and what you then did about it.
+     *
+     * EVERY REPLAY IS A POINT, including the ones with no numbers on them. A
+     * series that silently dropped the runs where the benchmark could not be
+     * scored would read as an unbroken line and not be one, so those points come
+     * back with `candidate_score` null and `status_reason` saying why.
+     */
+    async getBenchmarkHistory(id, params = {}) {
+        const q = new URLSearchParams();
+        if (params.limit != null)
+            q.set('limit', String(params.limit));
+        if (params.offset != null)
+            q.set('offset', String(params.offset));
+        const qs = q.toString();
+        return this._http.fetchGet(`/api/loop/benchmarks/${encodeURIComponent(id)}/history${qs ? `?${qs}` : ''}`);
+    }
+    /**
+     * Stop replaying a benchmark, and detach it from every rule that names it.
+     *
+     * Nothing measured is removed. There is no delete and no update on purpose:
+     * the series of numbers measured against a set is what a benchmark is for,
+     * so an edit would make everything before it incomparable with everything
+     * after it and a delete would throw the series away. Changing the set means
+     * pinning a new benchmark, and retiring the old one releases its name.
+     *
+     * Retiring twice is somebody pressing a button twice: the second call
+     * answers with the retired benchmark rather than a refusal.
+     */
+    async retireBenchmark(id, params = {}) {
+        const res = await this._http.fetchPost(`/api/loop/benchmarks/${encodeURIComponent(id)}/retire`, params);
+        return res.benchmark;
+    }
+    /**
+     * One replay in full: both absolute scores, the difference between them on
+     * the same set, the per-dimension breakdown and what it cost.
+     *
+     * Read `status` before you read the scores. A replay that did not score
+     * every pinned conversation publishes no score at all -- all three of
+     * `candidate_score`, `incumbent_score` and `score_delta` are null and
+     * `status_reason` is the sentence that explains it -- because a mean over
+     * whichever conversations happened to succeed is a measurement of a
+     * different set, which is the exact defect a standing benchmark removes.
+     */
+    async getBenchmarkRun(id) {
+        const res = await this._http.fetchGet(`/api/loop/benchmark-runs/${encodeURIComponent(id)}`);
+        return res.benchmark_run;
+    }
+    /**
+     * Attach a benchmark to a rule, or send `{ benchmark_id: null }` to detach
+     * it.
+     *
+     * Its own route rather than a field on the rule body, because it is a
+     * decision to replay a fixed set on every future run of this rule for as
+     * long as it stands, and its refusals -- retired, or belonging to another
+     * workspace -- are about the benchmark rather than about the rule.
+     *
+     * It does not invalidate consent and the reply says so: attaching raises
+     * neither the amount set aside for judge calls nor the amount set aside for
+     * keeping the new model available, so nobody is asked to read the same
+     * sentence again. A retired benchmark is refused with `BENCHMARK_RETIRED`,
+     * because a rule pointed at one would report no number on every run and say
+     * nothing about why.
+     */
+    async setTrainingRuleBenchmark(id, params) {
+        return this._http.fetchPut(`/api/loop/training-rules/${encodeURIComponent(id)}/benchmark`, params);
+    }
     // ── agent settings ────────────────────────────────────────────────────
     /**
      * The workspace's agent options: which model it defaults to, the system

package/dist/types.d.ts

@@ -3083,6 +3083,15 @@ export interface TrainingRule {
     eval_max_rows: number;
     eval_max_tokens: number;
     min_holdout_rows: number;
+    /**
+     * The standing benchmark replayed on every run of this rule, beside the
+     * per-run comparison and never instead of it. Null is the ordinary state.
+     *
+     * Set with `setTrainingRuleBenchmark`, not by an update to the rule: it is a
+     * decision to replay a fixed set on every future run, and it has refusals of
+     * its own. It raises no amount, so it does not invalidate consent.
+     */
+    benchmark_id: string | null;
     auto_promote: boolean;
     promote_margin: number;
     promote_min_win_rate: number;
@@ -3275,6 +3284,21 @@ export interface TrainingRulePreflightRequest {
 export interface TrainingRuleCreateRequest extends TrainingRulePreflightRequest {
     /** Absent is a refusal, not a default. */
     accept_terms?: TrainingRuleAcceptTerms;
+    /**
+     * The standing benchmark this rule replays, set here so it applies to the
+     * rule's FIRST run.
+     *
+     * A rule can fire within seconds of being created, and every step of a run
+     * reads the benchmark off the rule as it stood at that moment. Attaching one
+     * afterwards with `setTrainingRuleBenchmark` applies from the next run and
+     * says nothing about the first, which is the run somebody is watching.
+     *
+     * Refused on the same terms the attach route refuses it: `404` when it is not
+     * this workspace's, `409 BENCHMARK_RETIRED` when it has been retired. Use
+     * `setTrainingRuleBenchmark` to change or remove it later; it is not on the
+     * preflight body and not on the update body.
+     */
+    benchmark_id?: string;
 }
 export interface TrainingRuleUpdateRequest extends TrainingRulePreflightRequest {
     expected_revision?: number;
@@ -3352,6 +3376,15 @@ export interface TrainingRun {
     candidate_deadline_at: string | null;
     candidate_cleanup_at: string | null;
     evaluation_id: string | null;
+    /** This run's replay of the rule's standing benchmark, if it had one. */
+    benchmark_run_id: string | null;
+    /**
+     * What the replay's calls cost. Kept apart from `spent_eval_cents` so a
+     * report can say what the comparison cost and what the benchmark cost; both
+     * come out of `eval_ceiling_cents` and their sum can never exceed it, so
+     * anything totalling what a run cost has to add this one too.
+     */
+    benchmark_spent_cents: number;
     alias_name: string | null;
     alias_written_at: string | null;
     serving_before: TrainingRuleServing | null;
@@ -3389,6 +3422,8 @@ export interface TrainingRunSummary {
     billed_training_cents: number;
     billed_candidate_cents: number;
     spent_eval_cents: number;
+    /** The fourth money column. A row that leaves it out adds up short. */
+    benchmark_spent_cents: number;
     training_job_id: string | null;
     candidate_deployment_id: string | null;
     evaluation_id: string | null;
@@ -3414,6 +3449,8 @@ export interface TrainingRunLinks {
     candidate_url: string | null;
     dataset_url: string | null;
     evaluation_url: string | null;
+    /** The TREND the benchmark number belongs to, not the one replay. */
+    benchmark_url: string | null;
 }
 /** What this reader may do right now. A button that cannot work is never shown. */
 export interface TrainingRunActions {
@@ -3614,6 +3651,203 @@ export interface InferenceAliasRequest {
     target_inference_id: string;
     origin?: string;
 }
+/**
+ * A benchmark is active until it is retired. There is no delete and no update:
+ * the series of numbers measured against a set is what a benchmark is for, so
+ * an edit would make every number before it incomparable with every number
+ * after it, and a delete throws the series away.
+ */
+export type BenchmarkStatus = 'active' | 'retired';
+/** Where the pinned conversations are copied from. Read once, at creation. */
+export type BenchmarkSourceKind = 'traces' | 'dataset';
+/**
+ * The life of one replay.
+ *
+ * `budget_stopped` reached the amount left for it inside the run's own ceiling,
+ * and `abandoned` ran out of the time the run sets aside for it. Both are
+ * points on the trend carrying `status_reason`, never silent gaps.
+ */
+export type BenchmarkRunStatus = 'open' | 'done' | 'failed' | 'budget_stopped' | 'abandoned';
+/**
+ * How the pinned conversations become one number, frozen on the benchmark.
+ *
+ * `require_all_rows` is what makes the number comparable at all: a mean over
+ * whichever conversations happened to succeed is a measurement of a different
+ * set, so a short replay publishes no score and says why. A partial benchmark
+ * is a missing number, never a lower one.
+ */
+export interface BenchmarkScoring {
+    metric: string;
+    scale: string;
+    aggregate: string;
+    require_all_rows: boolean;
+}
+/**
+ * One rubric dimension of one replay, for both models.
+ *
+ * Deliberately not `EvaluationDimensionScore`: these are absolute means on a
+ * fixed set and those are paired means on that run's own held-back rows.
+ */
+export interface BenchmarkDimensionScore {
+    dimension: string;
+    incumbent: number;
+    candidate: number;
+    delta: number;
+}
+/**
+ * The frozen definition: the oracle, the scoring rule and the fingerprint of
+ * the set. The conversations themselves are their own paged route.
+ */
+export interface Benchmark {
+    id: string;
+    workspace_id: string;
+    name: string;
+    status: BenchmarkStatus;
+    /** Provenance only: everything needed from the judge is copied below it. */
+    judge_id: string | null;
+    judge_name: string;
+    judge_model: string;
+    judge_instructions: string;
+    judge_dimensions: EvaluationJudgeDimension[];
+    judge_system_prompt: string | null;
+    decoding: EvaluationDecoding;
+    scoring: BenchmarkScoring;
+    item_count: number;
+    /** Recomputed from the rows and compared at the start of every replay. */
+    items_digest: string;
+    /** A cap INSIDE the run's own amount for judge calls, never an addition to it. */
+    per_run_ceiling_cents: number;
+    created_by: string;
+    created_at: string;
+    retired_at: string | null;
+    retired_by: string | null;
+}
+/**
+ * One pinned conversation.
+ *
+ * `source_trace_id` is a link and is allowed to go null: the prompt was copied
+ * at pin time, so retention removing the conversation takes away the ability to
+ * open it and takes away nothing else. Every number already measured stays
+ * exactly as comparable as it was.
+ */
+export interface BenchmarkItem {
+    id: number;
+    ordinal: number;
+    source_trace_id: string | null;
+    source_trace_url: string | null;
+    prompt: TrainingMessage[];
+    tools: unknown[] | null;
+}
+/**
+ * One replay: this benchmark, on this training run, against both models.
+ *
+ * IT DOES NOT GATE PROMOTION. The paired comparison applies the judge to both
+ * sides of one conversation in one pass, so most of the judge's variance
+ * cancels in its delta; an absolute mean carries that variance whole. What
+ * ships here is the number, the difference against what serves today on the
+ * same set, and the history. A person reads the trend.
+ */
+export interface BenchmarkRun {
+    id: string;
+    benchmark_id: string;
+    benchmark_name: string;
+    run_id: string;
+    workspace_id: string;
+    incumbent_ref: EvaluationRef;
+    candidate_ref: EvaluationRef;
+    /** The set actually scored, recorded beside the score. */
+    items_digest: string;
+    rows_total: number;
+    rows_scored: number;
+    rows_failed: number;
+    /** Absolute, on the frozen judge's scale, and null unless every row scored. */
+    candidate_score: number | null;
+    incumbent_score: number | null;
+    score_delta: number | null;
+    per_dimension: BenchmarkDimensionScore[];
+    prompt_tokens: number;
+    completion_tokens: number;
+    spent_cents: number;
+    ceiling_cents: number;
+    status: BenchmarkRunStatus;
+    /** The whole explanation when there is no score, so never empty on one. */
+    status_reason: string | null;
+    created_at: string;
+    finished_at: string | null;
+}
+/**
+ * One point on the trend line.
+ *
+ * It carries the checkpoint and what the person then decided, because a trend
+ * with no idea what changed between two points is a chart rather than an
+ * answer.
+ */
+export interface BenchmarkHistoryPoint {
+    benchmark_run_id: string;
+    run_id: string;
+    run_seq: number;
+    rule_id: string | null;
+    rule_name: string | null;
+    created_at: string;
+    candidate_score: number | null;
+    incumbent_score: number | null;
+    score_delta: number | null;
+    checkpoint_id: string | null;
+    decision: TrainingDecision | null;
+    status: BenchmarkRunStatus;
+    status_reason: string | null;
+}
+/** Where the conversations are copied FROM. Read once, at creation, never again. */
+export interface BenchmarkSource {
+    kind: BenchmarkSourceKind;
+    trace_ids?: string[];
+    dataset_id?: string;
+    split?: string;
+}
+/**
+ * The pin: 10 to 200 conversations, and a judge with a rubric to measure them.
+ *
+ * A conversation with nothing to ask a model is refused rather than skipped.
+ * Pinning 47 of the 50 somebody chose is the set being wrong from the first
+ * day, and they would never find out.
+ */
+export interface BenchmarkCreateParams {
+    name: string;
+    judge_id: string;
+    source: BenchmarkSource;
+    /** Absent uses the judge's own model, and absent that the workspace default. */
+    judge_model?: string;
+    /** What both models are given to answer in. A shorter answer is a different answer. */
+    max_tokens?: number;
+    per_run_ceiling_cents: number;
+}
+export interface BenchmarkListParams {
+    status?: BenchmarkStatus;
+    limit?: number;
+    offset?: number;
+}
+export interface BenchmarkItemListParams {
+    /** Capped at 100 by the service. */
+    limit?: number;
+    offset?: number;
+}
+export interface BenchmarkHistoryParams {
+    limit?: number;
+    offset?: number;
+}
+/** A reason is a courtesy here, not a requirement. */
+export interface BenchmarkRetireParams {
+    reason?: string;
+}
+/**
+ * Attach a benchmark to a rule, or detach it with a present null.
+ *
+ * Not optional, and not omittable: this route sets the field, so an absent key
+ * would be a request with nothing in it. Send the id to attach, null to detach.
+ */
+export interface TrainingRuleBenchmarkRequest {
+    benchmark_id: string | null;
+}
 export interface TrainingRuleListResponse {
     rules: TrainingRule[];
     total: number;
@@ -3633,6 +3867,13 @@ export interface TrainingRuleMutationResponse {
 export interface TrainingRuleDeleteResponse {
     deleted: boolean;
     rule_id: string;
+    /**
+     * The name this delete just spent. The delete is a soft delete and the name
+     * index carries no partial predicate, so the row goes on holding the name
+     * after it has left every list you can read, and no later rule in the
+     * workspace can be called that. There is no purge and no undelete.
+     */
+    retained_name: string;
     cancelled_run_id: string | null;
 }
 export interface TrainingRunListResponse {
@@ -3642,6 +3883,12 @@ export interface TrainingRunListResponse {
 export interface TrainingRunResponse {
     run: TrainingRun;
     timeline: TrainingRunEvent[];
+    /**
+     * The standing benchmark's two absolute numbers for this run, beside the
+     * paired verdict and never in place of it. Null when no benchmark is
+     * attached to the rule.
+     */
+    benchmark: BenchmarkRun | null;
     links: TrainingRunLinks;
     available_actions: TrainingRunActions;
 }
@@ -3666,3 +3913,17 @@ export interface InferenceAliasResponse {
 export interface InferenceAliasDeleteResponse {
     deleted: boolean;
 }
+export interface BenchmarkListResponse {
+    benchmarks: Benchmark[];
+    total: number;
+}
+export interface BenchmarkItemsResponse {
+    items: BenchmarkItem[];
+    total: number;
+}
+/** The trend, newest first. Every replay is a point, including the scoreless ones. */
+export interface BenchmarkHistoryResponse {
+    benchmark_id: string;
+    points: BenchmarkHistoryPoint[];
+    total: number;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "runbios-sdk",
-  "version": "0.2.1-dev.152",
+  "version": "0.2.1-dev.158",
   "description": "Official TypeScript SDK for the Run BiOS training and deployment platform API",
   "type": "module",
   "main": "./dist/index.js",