runbios-sdk 0.2.21 → 0.2.22-rc.308
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.js +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/resources/inference.js +1 -1
- package/dist/resources/loop.d.ts +38 -11
- package/dist/resources/loop.js +37 -7
- package/dist/resources/training.js +18 -0
- package/dist/types.d.ts +273 -25
- package/package.json +1 -1
package/dist/client.js
CHANGED
|
@@ -344,7 +344,7 @@ export class HttpClient {
|
|
|
344
344
|
constructor(config) {
|
|
345
345
|
// Default host stays api.runbios.ai for now; cutover to api.runbios.ai is
|
|
346
346
|
// planned once its DNS exists.
|
|
347
|
-
this.baseUrl = (config.baseUrl || envBaseUrl() || 'https://api.runbios.ai').replace(/\/+$/, '');
|
|
347
|
+
this.baseUrl = (config.baseUrl || envBaseUrl() || 'https://api-staging.runbios.ai').replace(/\/+$/, '');
|
|
348
348
|
this.apiKey = config.apiKey ?? envApiKey();
|
|
349
349
|
this.accessToken = config.accessToken;
|
|
350
350
|
this.orgId = config.orgId;
|
package/dist/index.d.ts
CHANGED
|
@@ -36,7 +36,7 @@ import { Loop } from './resources/loop.js';
|
|
|
36
36
|
* SDK version. Sent as part of the User-Agent header.
|
|
37
37
|
* Must match package.json "version" -- enforced by a contract test.
|
|
38
38
|
*/
|
|
39
|
-
export declare const VERSION = "0.2.
|
|
39
|
+
export declare const VERSION = "0.2.22-rc.308";
|
|
40
40
|
export declare class RunBiOS {
|
|
41
41
|
/** Search models, fetch configs, check adapter compatibility. */
|
|
42
42
|
readonly models: Models;
|
|
@@ -75,7 +75,7 @@ export { Training } from './resources/training.js';
|
|
|
75
75
|
export { Wallet } from './resources/wallet.js';
|
|
76
76
|
export { GPU, type GPURecommendation } from './resources/gpu.js';
|
|
77
77
|
export { Inference, validateChatRequest, parseSSE, buildInferenceRequest, contextSizingBasis, type ContextSizingBasis, CONTEXT_DEFAULT_CEILING, CONTEXT_EDITABLE_FLOOR, type ChatMessage, type FunctionTool, type ChatCompletionParams, type ChatCompletionResponse, type ChatCompletionChunk, type InferenceVerbParams, type CompletionParams, type EmbeddingParams, type RerankParams, type AnthropicMessageParams, type ServerlessUsageWindow, type ServerlessTimeseriesMetric, type ServerlessWorkspaceLimits, type ServerlessUsageEnvelope, type ServerlessUsageOverviewResponse, type ServerlessUsageRowsResponse, type ServerlessUsageTimeseriesResponse, type ServerlessUsageDailyResponse, type ServerlessSavingsTotals, type ServerlessSavingsPeriod, type ServerlessUsageSavingsResponse, } from './resources/inference.js';
|
|
78
|
-
export type { BiOSConfig, PaginatedResponse, ApiErrorBody, AvailableGpuAlternative, CapacityMinimumRequirement, InferenceBookingAccepted, Model, ModelSearchParams, ModelSearchResponse, ModelDetailResponse, InferenceModel, InferenceModelListResponse, ModelConfig, Dataset, DatasetListParams, DatasetListResponse, DatasetUploadParams, DatasetPreview, DatasetPreviewParams, DatasetImportHFParams, DatasetRegisterHFParams, DatasetHubSearchParams, DatasetHubPreviewParams, DatasetValidation, DatasetFormatVariant, DatasetFormatSpec, DatasetFormatSpecs, DatasetStorageUsage, TrainingMethod, RLHFAlgorithm, AdapterType, TrainingJobStatus, TrainingStatusFilter, TrainingCreateParams, TrainingListParams, TrainingListResponse, TrainingJob, TrainingMetrics, TrainingResourceMetrics, TrainingDeviceMetrics, MetricPoint, MetricGraphConfig, TrainingCheckpoint, TrainingLogs, TrainingLogEntry, TrainingStopResponse, TrainingResumeResponse, CanonicalTrainingRequest, TrainingPreflightDataset, TrainingPreflightWarning, TrainingPreflightResponse, TrainingCapabilityChoice, TrainingConfigFieldCapability, TrainingCapabilities, GPUChoice, WalletBalance, Transaction, TransactionListResponse, TransactionListParams, GPUInfo, GPUPricingResponse, GPUOptionsParams, TrainingRecommendParams, TrainingAdvisorVerdict, TrainingAdvisorRecommendation, TrainingAdvisorUserShape, TrainingAdvisorJustification, TrainingAdvisorBasis, GPUOption, GPUOptionSuggestion, GPUOptionsResponse, InferenceStatus, InferenceCreateParams, InferenceUpdateParams, InferenceDeployment, InferenceDeploymentSummary, InferenceCreateResponse, InferenceListResponse, InferenceUpdateResponse, InferenceLifecycleResponse, InferenceDeleteResponse, InferenceGPUOptionMarket, InferenceGPUOption, InferenceGPUAlternative, InferenceGPUOptionsParams, InferenceGPUOptionsResponse, InferenceCanonicalRequest, InferencePreflightWarning, InferencePreflightResponse, AdapterCompatibility, AdapterCompatibilityResponse, AdapterCompatibilityParams, SupportedArchitecture, ArchitectureScope, SupportedArchitecturesResponse, ApiKey, ApiKeyScope, ApiKeyIntrospection, Organization, OrgMember, OrgInvite, Workspace, Integration, IntegrationCreateParams, StorageUsage, StorageObject, PromptTokensDetails, ChatCompletionUsage, ServingKind, TrainingRulePausedReason, TrainingTrigger, TrainingRecipe, TrainingRunState, TrainingVerdict, TrainingDecision, EvaluationStatus, EvaluationItemStatus, ItemErrorCode, EvaluationWinner, EvaluationItemWinnerFilter, ReportStatusFilter, ReportSort, EvaluationWarningCode, TrainingToolCall, TrainingMessage, TrainingRuleServing, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, TrainingRunListParams, TrainingRun, TrainingRunSummary, TrainingRunEvent, TrainingRunLinks, TrainingRunActions, EvaluationRef, EvaluationDecoding, EvaluationJudgeDimension, EvaluationDimensionScore, EvaluationWarning, EvaluationMargin, Evaluation, EvaluationItemJudge, EvaluationReferenceKind, EvaluationReference, EvaluationPairwiseOrder, EvaluationItemPairwise, EvaluationItemSide, EvaluationItem, EvaluationItemListParams, BenchmarkStatus, BenchmarkSourceKind, BenchmarkRunStatus, BenchmarkScoring, BenchmarkCheck, BenchmarkDimensionScore, Benchmark, BenchmarkItem, BenchmarkRun, BenchmarkRunItemSide, BenchmarkRunItem, BenchmarkRunItemListParams, BenchmarkHistoryPoint, BenchmarkSource, BenchmarkCreateParams, BenchmarkListParams, BenchmarkItemListParams, BenchmarkHistoryParams, BenchmarkRetireParams, InferenceAlias, InferenceAliasRequest, TrainingRuleDeleteResponse, TrainingRunListResponse, TrainingRunBenchmarkRun, TrainingRunResponse, TrainingRunActionResponse, EvaluationItemsResponse, BenchmarkRunItemsResponse, InferenceAliasListResponse, InferenceAliasResponse, InferenceAliasDeleteResponse, BenchmarkListResponse, BenchmarkItemsResponse, BenchmarkHistoryResponse, TrainingRuleRunRefusalCode, TrainingRuleSaveRefusalCode, TrainingRuleMonthlyLimitRefusal, PipelineStatus, PipelineActiveRun, PipelineVersion, PipelineAttempt, PipelineAttemptTrigger, PipelineAttemptOutcome, PipelineLiveDeploymentStatus, PipelineLiveDeployment, LoopStorageWhere, LoopStorage, Pipeline, PipelineListParams, PipelineListResponse, PipelineResponse, PipelineTrainWhen, PipelineSchedule, PipelineWriteRequest, PipelineMutationResponse, PipelineImportParams, PipelineFloors, HeldBackFloors, LoopModel, LoopModelsResponse, LoopTracePipeline, LoopTracePipelinesResponse, LoopMetricsHistoryParams, LoopMetricPoint, LoopSignalDay, LoopMetricsHistory, PromotionPolicyState, RuleBenchmark, PipelineBenchmarksResponse, PipelineBenchmarkAttachParams, PipelineBenchmarkMoveParams, CompareBenchmark, VersionBenchmarkScore, PromotionMeasurement, PromotionOutcome, VersionHoldout, VersionScores, VersionScoreDifference, VersionCompareResponse, VersionCompareParams, TrainingRunPromotion, VerdictCheck, VerdictCheckKey, VerdictChecksWithheld, TrainingRunCost, AttemptDirection, AttemptHoldout, AttemptCompareSide, AttemptSharedSample, AttemptSharedFigures, AttemptBenchmarkSide, AttemptCompareBenchmark, AttemptCompareItemSide, AttemptCompareItem, AttemptCompareResponse, AttemptCompareParams, } from './types.js';
|
|
78
|
+
export type { BiOSConfig, PaginatedResponse, ApiErrorBody, AvailableGpuAlternative, CapacityMinimumRequirement, InferenceBookingAccepted, Model, ModelSearchParams, ModelSearchResponse, ModelDetailResponse, InferenceModel, InferenceModelListResponse, ModelConfig, Dataset, DatasetListParams, DatasetListResponse, DatasetUploadParams, DatasetPreview, DatasetPreviewParams, DatasetImportHFParams, DatasetRegisterHFParams, DatasetHubSearchParams, DatasetHubPreviewParams, DatasetValidation, DatasetFormatVariant, DatasetFormatSpec, DatasetFormatSpecs, DatasetStorageUsage, TrainingMethod, RLHFAlgorithm, AdapterType, TrainingJobStatus, TrainingStatusFilter, TrainingCreateParams, TrainingStartFromParams, TrainingStartFrom, TrainingListParams, TrainingListResponse, TrainingJob, TrainingMetrics, TrainingResourceMetrics, TrainingDeviceMetrics, MetricPoint, MetricGraphConfig, TrainingCheckpoint, TrainingLogs, TrainingLogEntry, TrainingStopResponse, TrainingResumeResponse, CanonicalTrainingRequest, TrainingPreflightDataset, TrainingPreflightWarning, TrainingPreflightResponse, TrainingCapabilityChoice, TrainingConfigFieldCapability, TrainingCapabilities, TrainingFeature, GPUChoice, WalletBalance, Transaction, TransactionListResponse, TransactionListParams, GPUInfo, GPUPricingResponse, GPUOptionsParams, TrainingRecommendParams, TrainingAdvisorVerdict, TrainingAdvisorRecommendation, TrainingAdvisorUserShape, TrainingAdvisorJustification, TrainingAdvisorBasis, GPUOption, GPUOptionSuggestion, GPUOptionsResponse, InferenceStatus, InferenceCreateParams, InferenceUpdateParams, InferenceDeployment, InferenceDeploymentSummary, InferenceCreateResponse, InferenceListResponse, InferenceUpdateResponse, InferenceLifecycleResponse, InferenceDeleteResponse, InferenceGPUOptionMarket, InferenceGPUOption, InferenceGPUAlternative, InferenceGPUOptionsParams, InferenceGPUOptionsResponse, InferenceCanonicalRequest, InferencePreflightWarning, InferencePreflightResponse, AdapterCompatibility, AdapterCompatibilityResponse, AdapterCompatibilityParams, SupportedArchitecture, ArchitectureScope, SupportedArchitecturesResponse, ApiKey, ApiKeyScope, ApiKeyIntrospection, Organization, OrgMember, OrgInvite, Workspace, Integration, IntegrationCreateParams, StorageUsage, StorageObject, PromptTokensDetails, ChatCompletionUsage, ServingKind, TrainingRulePausedReason, TrainingTrigger, TrainingRecipe, TrainingRunState, TrainingVerdict, TrainingDecision, EvaluationStatus, EvaluationItemStatus, ItemErrorCode, EvaluationWinner, EvaluationItemWinnerFilter, ReportStatusFilter, ReportSort, EvaluationWarningCode, TrainingToolCall, TrainingMessage, TrainingRuleServing, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, TrainingRunListParams, TrainingRun, TrainingRunSummary, TrainingRunEvent, TrainingRunLinks, TrainingRunActions, EvaluationRef, EvaluationDecoding, EvaluationJudgeDimension, EvaluationDimensionScore, EvaluationWarning, EvaluationMargin, Evaluation, EvaluationItemJudge, EvaluationReferenceKind, EvaluationReference, EvaluationPairwiseOrder, EvaluationItemPairwise, EvaluationItemSide, EvaluationItem, EvaluationItemListParams, BenchmarkStatus, BenchmarkSourceKind, BenchmarkRunStatus, BenchmarkScoring, BenchmarkCheck, BenchmarkDimensionScore, Benchmark, BenchmarkItem, BenchmarkRun, BenchmarkRunItemSide, BenchmarkRunItem, BenchmarkRunItemListParams, BenchmarkHistoryPoint, BenchmarkSource, BenchmarkCreateParams, BenchmarkListParams, BenchmarkItemListParams, BenchmarkHistoryParams, BenchmarkRetireParams, InferenceAlias, InferenceAliasRequest, TrainingRuleDeleteResponse, TrainingRunListResponse, TrainingRunBenchmarkRun, TrainingRunResponse, TrainingRunActionResponse, EvaluationItemsResponse, BenchmarkRunItemsResponse, InferenceAliasListResponse, InferenceAliasResponse, InferenceAliasDeleteResponse, BenchmarkListResponse, BenchmarkItemsResponse, BenchmarkHistoryResponse, TrainingRuleRunRefusalCode, TrainingRuleSaveRefusalCode, TrainingRuleMonthlyLimitRefusal, PipelineStatus, PipelineActiveRun, PipelineVersion, PipelineAttempt, PipelineAttemptTrigger, Lineage, PipelineChain, PipelineChainRebuild, TrainingRunRollbackRefusalCode, LoopTraceRebuild, LoopTraceDeleteResponse, PipelineAttemptOutcome, PipelineLiveDeploymentStatus, PipelineLiveDeployment, LoopStorageWhere, LoopStorage, Pipeline, PipelineListParams, PipelineListResponse, PipelineResponse, PipelineTrainWhen, PipelineSchedule, PipelineWriteRequest, PipelineMutationResponse, PipelineImportParams, PipelineFloors, HeldBackFloors, LoopModel, LoopModelsResponse, LoopTracePipeline, LoopTracePipelinesResponse, LoopMetricsHistoryParams, LoopMetricPoint, LoopSignalDay, LoopMetricsHistory, PromotionPolicyState, RuleBenchmark, PipelineBenchmarksResponse, PipelineBenchmarkAttachParams, PipelineBenchmarkMoveParams, CompareBenchmark, VersionBenchmarkScore, PromotionMeasurement, PromotionOutcome, VersionHoldout, VersionScores, VersionScoreDifference, VersionCompareResponse, VersionCompareParams, TrainingRunPromotion, VerdictCheck, VerdictCheckKey, VerdictChecksWithheld, TrainingRunCost, AttemptDirection, AttemptHoldout, AttemptCompareSide, AttemptSharedSample, AttemptSharedFigures, AttemptBenchmarkSide, AttemptCompareBenchmark, AttemptCompareItemSide, AttemptCompareItem, AttemptCompareResponse, AttemptCompareParams, } from './types.js';
|
|
79
79
|
/** The closed set of run states a training run never leaves. */
|
|
80
80
|
export { TERMINAL_RUN_STATES } from './types.js';
|
|
81
81
|
/** The most versions a pipeline may be set to make. */
|
package/dist/index.js
CHANGED
|
@@ -36,7 +36,7 @@ import { Loop } from './resources/loop.js';
|
|
|
36
36
|
* SDK version. Sent as part of the User-Agent header.
|
|
37
37
|
* Must match package.json "version" -- enforced by a contract test.
|
|
38
38
|
*/
|
|
39
|
-
export const VERSION = '0.2.
|
|
39
|
+
export const VERSION = '0.2.22-rc.308';
|
|
40
40
|
export class RunBiOS {
|
|
41
41
|
/** Search models, fetch configs, check adapter compatibility. */
|
|
42
42
|
models;
|
|
@@ -336,7 +336,7 @@ export class Inference {
|
|
|
336
336
|
this.key = config.inferenceKey || envInferenceKey() || envApiKey();
|
|
337
337
|
// Same default host as HttpClient (api.runbios.ai); api.runbios.ai cutover
|
|
338
338
|
// is planned once its DNS exists — update both call sites together.
|
|
339
|
-
this.baseUrl = (config.baseUrl || envBaseUrl() || 'https://api.runbios.ai').replace(/\/+$/, '');
|
|
339
|
+
this.baseUrl = (config.baseUrl || envBaseUrl() || 'https://api-staging.runbios.ai').replace(/\/+$/, '');
|
|
340
340
|
this.timeout = config.timeout ?? 900_000;
|
|
341
341
|
this._http = http;
|
|
342
342
|
}
|
package/dist/resources/loop.d.ts
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
import type { HttpClient } from '../client.js';
|
|
2
|
-
import type { LoopTrace, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopImportParams, LoopImportResult, LoopSignal, LoopSignalParams, LoopLabel, LoopLabelCount, LoopConfig, LoopConfigParams, LoopStats, LoopTracePipelinesResponse, LoopMetricsHistory, LoopMetricsHistoryParams, TrainingRuleDeleteResponse, TrainingRunListParams, TrainingRunListResponse, TrainingRunResponse, TrainingRunActionResponse, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, Evaluation, EvaluationItemListParams, EvaluationItemsResponse, Benchmark, BenchmarkCreateParams, BenchmarkHistoryParams, BenchmarkHistoryResponse, BenchmarkItemListParams, BenchmarkItemsResponse, BenchmarkListParams, BenchmarkListResponse, BenchmarkRetireParams, BenchmarkRun, BenchmarkRunItemListParams, BenchmarkRunItemsResponse, Pipeline, PipelineListParams, PipelineListResponse, PipelineWriteRequest, PipelineMutationResponse, LoopModelsResponse, PipelineImportParams, PipelineBenchmarksResponse, PipelineBenchmarkAttachParams, PipelineBenchmarkMoveParams, VersionCompareResponse, VersionCompareParams, AttemptCompareResponse, AttemptCompareParams } from '../types.js';
|
|
2
|
+
import type { LoopTrace, LoopTraceDeleteResponse, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopImportParams, LoopImportResult, LoopSignal, LoopSignalParams, LoopLabel, LoopLabelCount, LoopConfig, LoopConfigParams, LoopStats, LoopTracePipelinesResponse, LoopMetricsHistory, LoopMetricsHistoryParams, TrainingRuleDeleteResponse, TrainingRunListParams, TrainingRunListResponse, TrainingRunResponse, TrainingRunActionResponse, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, Evaluation, EvaluationItemListParams, EvaluationItemsResponse, Benchmark, BenchmarkCreateParams, BenchmarkHistoryParams, BenchmarkHistoryResponse, BenchmarkItemListParams, BenchmarkItemsResponse, BenchmarkListParams, BenchmarkListResponse, BenchmarkRetireParams, BenchmarkRun, BenchmarkRunItemListParams, BenchmarkRunItemsResponse, Pipeline, PipelineListParams, PipelineListResponse, PipelineWriteRequest, PipelineMutationResponse, LoopModelsResponse, PipelineImportParams, PipelineBenchmarksResponse, PipelineBenchmarkAttachParams, PipelineBenchmarkMoveParams, VersionCompareResponse, VersionCompareParams, AttemptCompareResponse, AttemptCompareParams } from '../types.js';
|
|
3
3
|
/**
|
|
4
4
|
* The Conscious Loop -- pipelines that keep improving a model from the
|
|
5
5
|
* conversations you send them.
|
|
6
6
|
*
|
|
7
7
|
* A pipeline is one use case: a model, the tags whose samples train it, when
|
|
8
8
|
* to train, and how a new version is promoted. Every time it trains it makes
|
|
9
|
-
* an attempt; an attempt that
|
|
9
|
+
* an attempt; an attempt that goes live becomes the next version: one that
|
|
10
|
+
* beats the live model, one a member approves, or, after a deleted
|
|
11
|
+
* conversation, a rebuild without it that is not clearly worse (a `manual`
|
|
12
|
+
* pipeline waits for a member even then).
|
|
10
13
|
*
|
|
11
14
|
* Nothing is captured until a source is switched on, and you can switch it
|
|
12
15
|
* off again at any time. Creating a pipeline switches on the source named by
|
|
@@ -152,13 +155,16 @@ export declare class Loop {
|
|
|
152
155
|
*
|
|
153
156
|
* It also leaves every training set built from it, including sets already
|
|
154
157
|
* exported -- so a customer asking you to delete a conversation genuinely
|
|
155
|
-
* removes it from what the model will learn next.
|
|
156
|
-
*
|
|
158
|
+
* removes it from what the model will learn next. Where the live version of
|
|
159
|
+
* a pipeline learned from it, that pipeline rebuilds its model from the
|
|
160
|
+
* base on every remaining sample, without it: `rebuilds` names each one.
|
|
161
|
+
* Until the rebuild goes live, the live version still holds what it
|
|
162
|
+
* learned; once it does, no version that learned the conversation can be
|
|
163
|
+
* rolled back to, and their stored weights are deleted. Other finished runs
|
|
164
|
+
* are not affected, and a conversation that expires under the retention
|
|
165
|
+
* setting rebuilds nothing.
|
|
157
166
|
*/
|
|
158
|
-
deleteTrace(id: string): Promise<
|
|
159
|
-
deleted: boolean;
|
|
160
|
-
trace_id: string;
|
|
161
|
-
}>;
|
|
167
|
+
deleteTrace(id: string): Promise<LoopTraceDeleteResponse>;
|
|
162
168
|
/**
|
|
163
169
|
* Record a verdict on a conversation.
|
|
164
170
|
*
|
|
@@ -334,6 +340,14 @@ export declare class Loop {
|
|
|
334
340
|
* The reply carries `serving`: this is the one action besides promotion that
|
|
335
341
|
* changes what answers your traffic, so what it was put back to is returned
|
|
336
342
|
* rather than left to be looked up.
|
|
343
|
+
*
|
|
344
|
+
* Where the pipeline builds on its live version, the version put back is
|
|
345
|
+
* what the next attempt builds on, and it trains again on what the
|
|
346
|
+
* rolled-back version learned. Refusals are a
|
|
347
|
+
* `TrainingRunRollbackRefusalCode`: `RUN_ACTIVE` also while an attempt is
|
|
348
|
+
* being built on this version (cancel it with {@link cancelTrainingRun}
|
|
349
|
+
* first), and `ROLLBACK_DELETED_DATA` when the version it would put back
|
|
350
|
+
* learned from a conversation that was deleted since.
|
|
337
351
|
*/
|
|
338
352
|
rollbackTrainingRun(id: string, params?: TrainingRunRollbackRequest): Promise<TrainingRunActionResponse>;
|
|
339
353
|
/**
|
|
@@ -370,11 +384,16 @@ export declare class Loop {
|
|
|
370
384
|
* The one list {@link createPipeline} accepts a `model.id` from, with the
|
|
371
385
|
* fixed `floors` a pipeline is held to: the samples its first attempt
|
|
372
386
|
* needs, and the new samples each later attempt waits for
|
|
373
|
-
* (`new_samples`,
|
|
374
|
-
* attempt trained on
|
|
387
|
+
* (`new_samples`, counted since the last trained attempt, however many
|
|
388
|
+
* rows that attempt trained on), and the held-back rule
|
|
375
389
|
* (`held_back`: at least `min_rows`, or the share a pipeline chooses from
|
|
376
390
|
* `default_percent` to `max_percent` if that is more, and each attempt is
|
|
377
391
|
* judged on at most `scored_rows` of them). They are read-only.
|
|
392
|
+
* Where a pipeline builds on its live version (its `chain` is set), each
|
|
393
|
+
* attempt trains a new adapter on top of the live version, on what it has
|
|
394
|
+
* not learned yet plus a replay of what it already learned (about a fifth
|
|
395
|
+
* of what it trains on, picked at random, so it is not forgotten), and a
|
|
396
|
+
* winner is merged in and served as one model.
|
|
378
397
|
* When the list cannot be read it throws `ApiError` 503
|
|
379
398
|
* `MODELS_UNAVAILABLE`, whose `body.floors` still carries the floors.
|
|
380
399
|
*/
|
|
@@ -386,6 +405,11 @@ export declare class Loop {
|
|
|
386
405
|
* whenever its data has grown enough (and at its schedule, when it has one),
|
|
387
406
|
* the platform trains a new version on GPUs it chooses, compares it with what
|
|
388
407
|
* serves your traffic, and promotes it by the pipeline's promotion policy.
|
|
408
|
+
* Where the pipeline builds on its live version (`chain`), each attempt
|
|
409
|
+
* waits for 1,000 new samples and trains a new adapter on top of the live
|
|
410
|
+
* version, on what it has not learned plus a replay of what it already
|
|
411
|
+
* learned (about a fifth of what it trains on, picked at random); a winner
|
|
412
|
+
* is merged in.
|
|
389
413
|
* What is charged is the GPU time the runs actually use; the platform's own
|
|
390
414
|
* per-run and monthly limits stop a run that goes wrong. `holdout.percent`
|
|
391
415
|
* (5 to 20, default 5) is the share of its samples held back to judge each
|
|
@@ -423,6 +447,8 @@ export declare class Loop {
|
|
|
423
447
|
* `TrainingRuleRunRefusalCode`. When the attempt in flight is waiting for
|
|
424
448
|
* GPUs to compare (`active_run.state` `waiting_gpus`), this books its two
|
|
425
449
|
* comparison machines again at once instead of starting another attempt.
|
|
450
|
+
* With a rebuild on its way (`chain.rebuild`), it starts the rebuild: from
|
|
451
|
+
* the base on every remaining sample, with no wait for new ones.
|
|
426
452
|
*/
|
|
427
453
|
runPipeline(id: string): Promise<PipelineMutationResponse>;
|
|
428
454
|
/**
|
|
@@ -495,7 +521,8 @@ export declare class Loop {
|
|
|
495
521
|
* `agreed` or `disagreed`), `has_reference`, `status` (`scored`, `failed`),
|
|
496
522
|
* order by `sort` (`score_change`: the attempt's score less the other's,
|
|
497
523
|
* largest first), and search with `q` (2 to 200 characters) over each
|
|
498
|
-
* conversation
|
|
524
|
+
* conversation (not the system prompt every row shares), its right answer,
|
|
525
|
+
* both answers and every reason; `searched`
|
|
499
526
|
* says how many rows were searched.
|
|
500
527
|
*/
|
|
501
528
|
listEvaluationItems(id: string, params?: EvaluationItemListParams): Promise<EvaluationItemsResponse>;
|
package/dist/resources/loop.js
CHANGED
|
@@ -4,7 +4,10 @@
|
|
|
4
4
|
*
|
|
5
5
|
* A pipeline is one use case: a model, the tags whose samples train it, when
|
|
6
6
|
* to train, and how a new version is promoted. Every time it trains it makes
|
|
7
|
-
* an attempt; an attempt that
|
|
7
|
+
* an attempt; an attempt that goes live becomes the next version: one that
|
|
8
|
+
* beats the live model, one a member approves, or, after a deleted
|
|
9
|
+
* conversation, a rebuild without it that is not clearly worse (a `manual`
|
|
10
|
+
* pipeline waits for a member even then).
|
|
8
11
|
*
|
|
9
12
|
* Nothing is captured until a source is switched on, and you can switch it
|
|
10
13
|
* off again at any time. Creating a pipeline switches on the source named by
|
|
@@ -199,8 +202,14 @@ export class Loop {
|
|
|
199
202
|
*
|
|
200
203
|
* It also leaves every training set built from it, including sets already
|
|
201
204
|
* exported -- so a customer asking you to delete a conversation genuinely
|
|
202
|
-
* removes it from what the model will learn next.
|
|
203
|
-
*
|
|
205
|
+
* removes it from what the model will learn next. Where the live version of
|
|
206
|
+
* a pipeline learned from it, that pipeline rebuilds its model from the
|
|
207
|
+
* base on every remaining sample, without it: `rebuilds` names each one.
|
|
208
|
+
* Until the rebuild goes live, the live version still holds what it
|
|
209
|
+
* learned; once it does, no version that learned the conversation can be
|
|
210
|
+
* rolled back to, and their stored weights are deleted. Other finished runs
|
|
211
|
+
* are not affected, and a conversation that expires under the retention
|
|
212
|
+
* setting rebuilds nothing.
|
|
204
213
|
*/
|
|
205
214
|
async deleteTrace(id) {
|
|
206
215
|
return this._http.fetchDelete(`/api/loop/traces/${encodeURIComponent(id)}`);
|
|
@@ -401,7 +410,7 @@ export class Loop {
|
|
|
401
410
|
// ── attempts ──────────────────────────────────────────────────────────
|
|
402
411
|
//
|
|
403
412
|
// Every time a pipeline trains it makes an attempt (a training run). The
|
|
404
|
-
// attempts that
|
|
413
|
+
// attempts that went live became its versions.
|
|
405
414
|
/** Attempts, newest first. Filter by pipeline (`rule_id`) or by the state they are sitting in. */
|
|
406
415
|
async listTrainingRuns(params = {}) {
|
|
407
416
|
const q = new URLSearchParams();
|
|
@@ -451,6 +460,14 @@ export class Loop {
|
|
|
451
460
|
* The reply carries `serving`: this is the one action besides promotion that
|
|
452
461
|
* changes what answers your traffic, so what it was put back to is returned
|
|
453
462
|
* rather than left to be looked up.
|
|
463
|
+
*
|
|
464
|
+
* Where the pipeline builds on its live version, the version put back is
|
|
465
|
+
* what the next attempt builds on, and it trains again on what the
|
|
466
|
+
* rolled-back version learned. Refusals are a
|
|
467
|
+
* `TrainingRunRollbackRefusalCode`: `RUN_ACTIVE` also while an attempt is
|
|
468
|
+
* being built on this version (cancel it with {@link cancelTrainingRun}
|
|
469
|
+
* first), and `ROLLBACK_DELETED_DATA` when the version it would put back
|
|
470
|
+
* learned from a conversation that was deleted since.
|
|
454
471
|
*/
|
|
455
472
|
async rollbackTrainingRun(id, params = {}) {
|
|
456
473
|
return this._http.fetchPost(`/api/loop/training-runs/${encodeURIComponent(id)}/rollback`, params);
|
|
@@ -514,11 +531,16 @@ export class Loop {
|
|
|
514
531
|
* The one list {@link createPipeline} accepts a `model.id` from, with the
|
|
515
532
|
* fixed `floors` a pipeline is held to: the samples its first attempt
|
|
516
533
|
* needs, and the new samples each later attempt waits for
|
|
517
|
-
* (`new_samples`,
|
|
518
|
-
* attempt trained on
|
|
534
|
+
* (`new_samples`, counted since the last trained attempt, however many
|
|
535
|
+
* rows that attempt trained on), and the held-back rule
|
|
519
536
|
* (`held_back`: at least `min_rows`, or the share a pipeline chooses from
|
|
520
537
|
* `default_percent` to `max_percent` if that is more, and each attempt is
|
|
521
538
|
* judged on at most `scored_rows` of them). They are read-only.
|
|
539
|
+
* Where a pipeline builds on its live version (its `chain` is set), each
|
|
540
|
+
* attempt trains a new adapter on top of the live version, on what it has
|
|
541
|
+
* not learned yet plus a replay of what it already learned (about a fifth
|
|
542
|
+
* of what it trains on, picked at random, so it is not forgotten), and a
|
|
543
|
+
* winner is merged in and served as one model.
|
|
522
544
|
* When the list cannot be read it throws `ApiError` 503
|
|
523
545
|
* `MODELS_UNAVAILABLE`, whose `body.floors` still carries the floors.
|
|
524
546
|
*/
|
|
@@ -533,6 +555,11 @@ export class Loop {
|
|
|
533
555
|
* whenever its data has grown enough (and at its schedule, when it has one),
|
|
534
556
|
* the platform trains a new version on GPUs it chooses, compares it with what
|
|
535
557
|
* serves your traffic, and promotes it by the pipeline's promotion policy.
|
|
558
|
+
* Where the pipeline builds on its live version (`chain`), each attempt
|
|
559
|
+
* waits for 1,000 new samples and trains a new adapter on top of the live
|
|
560
|
+
* version, on what it has not learned plus a replay of what it already
|
|
561
|
+
* learned (about a fifth of what it trains on, picked at random); a winner
|
|
562
|
+
* is merged in.
|
|
536
563
|
* What is charged is the GPU time the runs actually use; the platform's own
|
|
537
564
|
* per-run and monthly limits stop a run that goes wrong. `holdout.percent`
|
|
538
565
|
* (5 to 20, default 5) is the share of its samples held back to judge each
|
|
@@ -576,6 +603,8 @@ export class Loop {
|
|
|
576
603
|
* `TrainingRuleRunRefusalCode`. When the attempt in flight is waiting for
|
|
577
604
|
* GPUs to compare (`active_run.state` `waiting_gpus`), this books its two
|
|
578
605
|
* comparison machines again at once instead of starting another attempt.
|
|
606
|
+
* With a rebuild on its way (`chain.rebuild`), it starts the rebuild: from
|
|
607
|
+
* the base on every remaining sample, with no wait for new ones.
|
|
579
608
|
*/
|
|
580
609
|
async runPipeline(id) {
|
|
581
610
|
return this._http.fetchPost(`/api/loop/pipelines/${encodeURIComponent(id)}/run`, {});
|
|
@@ -683,7 +712,8 @@ export class Loop {
|
|
|
683
712
|
* `agreed` or `disagreed`), `has_reference`, `status` (`scored`, `failed`),
|
|
684
713
|
* order by `sort` (`score_change`: the attempt's score less the other's,
|
|
685
714
|
* largest first), and search with `q` (2 to 200 characters) over each
|
|
686
|
-
* conversation
|
|
715
|
+
* conversation (not the system prompt every row shares), its right answer,
|
|
716
|
+
* both answers and every reason; `searched`
|
|
687
717
|
* says how many rows were searched.
|
|
688
718
|
*/
|
|
689
719
|
async listEvaluationItems(id, params = {}) {
|
|
@@ -7,6 +7,22 @@ function trainingIdempotencyKey(explicit) {
|
|
|
7
7
|
}
|
|
8
8
|
return key;
|
|
9
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* `start_from` on the wire. The adapters stay in the order given (oldest
|
|
12
|
+
* first: the order they are merged in), and an empty list is refused here
|
|
13
|
+
* rather than sent, since the API refuses it too.
|
|
14
|
+
*/
|
|
15
|
+
function startFromBody(sf) {
|
|
16
|
+
if (!Array.isArray(sf.adapters) || sf.adapters.length === 0) {
|
|
17
|
+
throw new Error('RunBiOS: startFrom.adapters must name at least one adapter checkpoint, oldest first');
|
|
18
|
+
}
|
|
19
|
+
const body = {
|
|
20
|
+
adapters: sf.adapters.map((a) => ({ job_id: a.jobId, checkpoint_id: a.checkpointId })),
|
|
21
|
+
};
|
|
22
|
+
if (sf.exportMerged !== undefined)
|
|
23
|
+
body.export_merged = sf.exportMerged;
|
|
24
|
+
return body;
|
|
25
|
+
}
|
|
10
26
|
function buildTrainingRequest(params) {
|
|
11
27
|
let trainingMethod = params.method;
|
|
12
28
|
// Legacy alias from SDK versions that modeled VLM training as a method;
|
|
@@ -80,6 +96,8 @@ function buildTrainingRequest(params) {
|
|
|
80
96
|
body.dataset_mixing = params.datasetMixing;
|
|
81
97
|
if (params.mixing !== undefined)
|
|
82
98
|
body.mixing = params.mixing;
|
|
99
|
+
if (params.startFrom !== undefined)
|
|
100
|
+
body.start_from = startFromBody(params.startFrom);
|
|
83
101
|
const config = {};
|
|
84
102
|
if (params.epochs !== undefined)
|
|
85
103
|
config.num_train_epochs = params.epochs;
|
package/dist/types.d.ts
CHANGED
|
@@ -8,13 +8,13 @@ export interface BiOSConfig {
|
|
|
8
8
|
orgId?: string;
|
|
9
9
|
/** Workspace ID. Optional when using API keys (resolved from the key). Can override for multi-workspace keys. */
|
|
10
10
|
workspaceId?: string;
|
|
11
|
-
/** Base URL for the API. Falls back to the RUNBIOS_BASE_URL environment variable (legacy: BIOS_BASE_URL), then the canonical https://api.runbios.ai hostname. */
|
|
11
|
+
/** Base URL for the API. Falls back to the RUNBIOS_BASE_URL environment variable (legacy: BIOS_BASE_URL), then the canonical https://api-staging.runbios.ai hostname. */
|
|
12
12
|
baseUrl?: string;
|
|
13
13
|
/** Request timeout in milliseconds. Defaults to 30000. */
|
|
14
14
|
timeout?: number;
|
|
15
15
|
/** Default per-deployment inference key. Can be overridden per inference call. */
|
|
16
16
|
inferenceKey?: string;
|
|
17
|
-
/** Inference base URL. Defaults to baseUrl, then https://api.runbios.ai. */
|
|
17
|
+
/** Inference base URL. Defaults to baseUrl, then https://api-staging.runbios.ai. */
|
|
18
18
|
inferenceBaseUrl?: string;
|
|
19
19
|
/** End-to-end inference timeout in milliseconds. Defaults to 15 minutes. */
|
|
20
20
|
inferenceTimeout?: number;
|
|
@@ -547,15 +547,60 @@ export interface GPUChoice {
|
|
|
547
547
|
/** Native market tier; training and deployment capacity are secure-only. */
|
|
548
548
|
tier?: 'secure';
|
|
549
549
|
}
|
|
550
|
+
/**
|
|
551
|
+
* Train a new LoRA adapter on top of earlier LoRA versions of the same base
|
|
552
|
+
* model (`start_from` on create and preflight). Feature-gated: only when
|
|
553
|
+
* `training.capabilities()` lists the `start_from_adapters` feature as
|
|
554
|
+
* enabled; otherwise the API answers 400 TRAINING_FEATURE_DISABLED with the
|
|
555
|
+
* feature's reason.
|
|
556
|
+
*
|
|
557
|
+
* `adapters` are 1 to 64 adapter checkpoints of earlier jobs, OLDEST FIRST:
|
|
558
|
+
* the job trains on the base with every one of them merged into it in that
|
|
559
|
+
* order. Each must be an exact adapter checkpoint of a completed or stopped
|
|
560
|
+
* LoRA job in the same workspace, on the same pinned model and revision, and
|
|
561
|
+
* each earlier job must itself have started from exactly the adapters before
|
|
562
|
+
* it. The method must be sft or pt and the adapter lora.
|
|
563
|
+
*
|
|
564
|
+
* `exportMerged` also writes the base, the adapters and this job's selected
|
|
565
|
+
* adapter merged into one model: a checkpoint of format `merged`, which is
|
|
566
|
+
* what deploys (the job's own adapters are not deployable alone).
|
|
567
|
+
*
|
|
568
|
+
* Refusals, all before anything is charged: 404 for an unknown, deleted or
|
|
569
|
+
* other-workspace checkpoint; 422 START_FROM_BASE_MISMATCH,
|
|
570
|
+
* START_FROM_CHAIN_BROKEN or START_FROM_LINK_UNREADABLE naming the `index`;
|
|
571
|
+
* 422 START_FROM_NOT_SUPPORTED (a model type or qlora that cannot chain).
|
|
572
|
+
* A job trained this way cannot be resumed (422
|
|
573
|
+
* START_FROM_RESUME_UNSUPPORTED): start it again. If the merged model cannot
|
|
574
|
+
* be saved after training, the job fails with MERGED_EXPORT_FAILED.
|
|
575
|
+
*/
|
|
576
|
+
export interface TrainingStartFromParams {
|
|
577
|
+
adapters: {
|
|
578
|
+
jobId: string;
|
|
579
|
+
checkpointId: string;
|
|
580
|
+
}[];
|
|
581
|
+
exportMerged?: boolean;
|
|
582
|
+
}
|
|
583
|
+
/** `start_from` as the API echoes it on create, preflight and the job, with the digest of the chain's exact bytes. */
|
|
584
|
+
export interface TrainingStartFrom {
|
|
585
|
+
adapters: {
|
|
586
|
+
job_id: string;
|
|
587
|
+
checkpoint_id: string;
|
|
588
|
+
}[];
|
|
589
|
+
export_merged: boolean;
|
|
590
|
+
chain_digest?: string;
|
|
591
|
+
}
|
|
550
592
|
/** Parameters for creating a training job. */
|
|
551
593
|
export interface TrainingCreateParams {
|
|
594
|
+
/** Train on top of earlier LoRA versions of the same base. See {@link TrainingStartFromParams}. */
|
|
595
|
+
startFrom?: TrainingStartFromParams;
|
|
552
596
|
/** Stable retry key. Reuse it after a timeout; a new value creates a new job. Auto-generated when omitted. */
|
|
553
597
|
idempotencyKey?: string;
|
|
554
598
|
/**
|
|
555
599
|
* Model id to fine-tune. Must be hosted on Run BiOS — pick from the platform's
|
|
556
600
|
* own catalog (client.models.search); arbitrary Hugging Face ids are
|
|
557
|
-
* rejected.
|
|
558
|
-
*
|
|
601
|
+
* rejected. Every job starts from a catalog base model; `startFrom` (where
|
|
602
|
+
* the platform enables it) trains a new LoRA adapter on top of earlier LoRA
|
|
603
|
+
* versions of that same base.
|
|
559
604
|
*/
|
|
560
605
|
model: string;
|
|
561
606
|
/** Optional model branch, tag, or commit; the API canonicalizes it to an exact commit. */
|
|
@@ -744,6 +789,10 @@ export interface TrainingJob {
|
|
|
744
789
|
created_at?: string;
|
|
745
790
|
billing_started_at?: string;
|
|
746
791
|
resume_from_job_id?: string | null;
|
|
792
|
+
/** The earlier LoRA versions this job trained on top of; absent for a job from the base. See {@link TrainingStartFromParams}. */
|
|
793
|
+
start_from?: TrainingStartFrom;
|
|
794
|
+
/** The job's merged model (a checkpoint of format `merged`) once it is saved; null until then. Only on a job with `start_from`. */
|
|
795
|
+
merged_checkpoint_id?: string | null;
|
|
747
796
|
resume_count?: number;
|
|
748
797
|
can_resume?: boolean;
|
|
749
798
|
has_cached_dataset?: boolean;
|
|
@@ -979,6 +1028,17 @@ export interface TrainingCheckpoint {
|
|
|
979
1028
|
base_model_id: string;
|
|
980
1029
|
base_model_revision: string;
|
|
981
1030
|
created_at: string;
|
|
1031
|
+
/**
|
|
1032
|
+
* What the stored weights are: `adapter`, `full`, or `merged` -- the base
|
|
1033
|
+
* with the job's `start_from` adapters and its selected adapter merged into
|
|
1034
|
+
* one model, named "merged model (step B)", with `step` the adapter step it
|
|
1035
|
+
* was merged from. A merged model is never final, best or pruned, and is
|
|
1036
|
+
* what deploys: an adapter of a job with `start_from` is refused by
|
|
1037
|
+
* deployments (CHAINED_ADAPTER_NOT_DEPLOYABLE). Absent on an older service.
|
|
1038
|
+
*/
|
|
1039
|
+
format?: 'adapter' | 'full' | 'merged';
|
|
1040
|
+
/** On a merged model: the adapter checkpoint it was merged from. */
|
|
1041
|
+
merged_of_checkpoint_id?: string;
|
|
982
1042
|
}
|
|
983
1043
|
/** Training logs response. */
|
|
984
1044
|
export interface TrainingLogs {
|
|
@@ -1245,6 +1305,24 @@ export interface TrainingConfigFieldCapability {
|
|
|
1245
1305
|
adapters?: string[];
|
|
1246
1306
|
description?: string;
|
|
1247
1307
|
}
|
|
1308
|
+
/**
|
|
1309
|
+
* A whole capability of GET /api/training/capabilities that is on or off as a
|
|
1310
|
+
* unit (`features`), such as `start_from_adapters` (training on top of
|
|
1311
|
+
* earlier LoRA versions, `start_from`). Check `enabled` before using it; when
|
|
1312
|
+
* it is off, `disabled_reason` says why, in words for a person.
|
|
1313
|
+
*/
|
|
1314
|
+
export interface TrainingFeature {
|
|
1315
|
+
id: string;
|
|
1316
|
+
name: string;
|
|
1317
|
+
enabled: boolean;
|
|
1318
|
+
disabled_reason?: string;
|
|
1319
|
+
/** The adapter types it works with, e.g. `["lora"]`. */
|
|
1320
|
+
train_types?: string[];
|
|
1321
|
+
/** The most adapters one `start_from` may name. */
|
|
1322
|
+
max_adapters?: number;
|
|
1323
|
+
/** Model types it cannot be used with. */
|
|
1324
|
+
unsupported_model_types?: string[];
|
|
1325
|
+
}
|
|
1248
1326
|
/** Dynamic contract returned by GET /api/training/capabilities. */
|
|
1249
1327
|
export interface TrainingCapabilities {
|
|
1250
1328
|
contract_version: string;
|
|
@@ -1258,6 +1336,14 @@ export interface TrainingCapabilities {
|
|
|
1258
1336
|
when: Record<string, unknown>;
|
|
1259
1337
|
message: string;
|
|
1260
1338
|
}>;
|
|
1339
|
+
/**
|
|
1340
|
+
* Whole capabilities that are on or off as a unit, such as
|
|
1341
|
+
* `start_from_adapters`. Optional: an older training service does not send
|
|
1342
|
+
* it, and a feature it does not list is not available.
|
|
1343
|
+
*/
|
|
1344
|
+
features?: TrainingFeature[];
|
|
1345
|
+
/** The training engine's native Hugging Face architecture classes, when the service sends them. */
|
|
1346
|
+
architectures?: string[];
|
|
1261
1347
|
}
|
|
1262
1348
|
/**
|
|
1263
1349
|
* Wallet balance (`GET /api/billing/wallet`).
|
|
@@ -2869,11 +2955,61 @@ export type TrainingRulePausedReason = 'consent_invalid' | 'insufficient_funds'
|
|
|
2869
2955
|
/**
|
|
2870
2956
|
* Why a run fired, as the run stores it: `rows` (enough new samples),
|
|
2871
2957
|
* `cadence` (the schedule), `both`, `manual` (train now), `challenger` (the
|
|
2872
|
-
* pipeline's challenger model trained
|
|
2873
|
-
* it),
|
|
2874
|
-
*
|
|
2958
|
+
* pipeline's challenger model, trained from its own base on every sample the
|
|
2959
|
+
* attempt before it could train on), `rebuild` (a member deleted a
|
|
2960
|
+
* conversation the live version learned from, so the model is trained again
|
|
2961
|
+
* from the base on every remaining sample), or `variant` (a recipe variant,
|
|
2962
|
+
* retired: the same base model on the same conversations with one training
|
|
2963
|
+
* setting changed; old runs only).
|
|
2964
|
+
*/
|
|
2965
|
+
export type TrainingTrigger = 'rows' | 'cadence' | 'both' | 'manual' | 'variant' | 'challenger' | 'rebuild';
|
|
2966
|
+
/**
|
|
2967
|
+
* What an attempt was trained on top of, and on which samples: `lineage` on a
|
|
2968
|
+
* run, on a pipeline's versions, attempts and attempt in flight, and on each
|
|
2969
|
+
* side of an attempt comparison.
|
|
2970
|
+
*
|
|
2971
|
+
* `trained_from` is `versions` when the attempt trained a new adapter on top
|
|
2972
|
+
* of the live version: the pipeline's pinned base with the versions in
|
|
2973
|
+
* `built_on` (oldest first) merged into it in that order. It is `base` when
|
|
2974
|
+
* it started from the base model, and `from_base_reason` says why:
|
|
2975
|
+
* `first_version`, `challenger` (the second model never builds on another
|
|
2976
|
+
* model's versions), `rebuild` (a deleted conversation, with `rebuild` true;
|
|
2977
|
+
* or, with `rebuild` false, a live version already built on 64 versions, the
|
|
2978
|
+
* most one attempt can build on), `model_cannot_build_on` or
|
|
2979
|
+
* `chain_link_unreadable` (an earlier version could not be read). Null when
|
|
2980
|
+
* it built on versions.
|
|
2981
|
+
*
|
|
2982
|
+
* What it trained on (`train_rows`) is `new_samples` (new since the attempt
|
|
2983
|
+
* before it) + `carried_samples` (trained by the attempts in
|
|
2984
|
+
* `carried_from_attempts`, which did not go live) + `replayed_samples` (about
|
|
2985
|
+
* a fifth of what it trained on, drawn at random from what the versions it
|
|
2986
|
+
* builds on already learned, so it does not forget them). `learned_before` is
|
|
2987
|
+
* how many samples those versions had learned: they are not trained again,
|
|
2988
|
+
* except the replayed ones. An attempt from the base trains on every sample
|
|
2989
|
+
* it can: `new_samples` is all of them, and the other three counts are 0.
|
|
2990
|
+
* The four counts are null on an attempt from before the chain and on every
|
|
2991
|
+
* attempt of a pipeline that does not build on its live version: only
|
|
2992
|
+
* `train_rows` is known then.
|
|
2993
|
+
*
|
|
2994
|
+
* `served_as` is `adapter` for an attempt from the base and `merged` for one
|
|
2995
|
+
* built on versions (one model with every version merged in). A winner is
|
|
2996
|
+
* merged in and served as one model. `learned_deleted_conversation` is true
|
|
2997
|
+
* when a conversation it trained on has since been deleted: a rebuild
|
|
2998
|
+
* without it is on its way, and it can no longer be rolled back to.
|
|
2875
2999
|
*/
|
|
2876
|
-
export
|
|
3000
|
+
export interface Lineage {
|
|
3001
|
+
trained_from: 'base' | 'versions';
|
|
3002
|
+
built_on: number[];
|
|
3003
|
+
served_as: 'adapter' | 'merged';
|
|
3004
|
+
new_samples: number | null;
|
|
3005
|
+
carried_samples: number | null;
|
|
3006
|
+
carried_from_attempts: number[];
|
|
3007
|
+
replayed_samples: number | null;
|
|
3008
|
+
learned_before: number | null;
|
|
3009
|
+
rebuild: boolean;
|
|
3010
|
+
learned_deleted_conversation: boolean;
|
|
3011
|
+
from_base_reason: 'first_version' | 'challenger' | 'rebuild' | 'model_cannot_build_on' | 'chain_link_unreadable' | null;
|
|
3012
|
+
}
|
|
2877
3013
|
/**
|
|
2878
3014
|
* The four training settings a recipe variant may change, as the run was
|
|
2879
3015
|
* actually trained: the training service's own defaults are filled in where
|
|
@@ -2968,6 +3104,39 @@ export interface TrainingRunRejectRequest {
|
|
|
2968
3104
|
export interface TrainingRunRollbackRequest {
|
|
2969
3105
|
reason?: string;
|
|
2970
3106
|
}
|
|
3107
|
+
/**
|
|
3108
|
+
* The machine codes `rollbackTrainingRun` can be refused with.
|
|
3109
|
+
*
|
|
3110
|
+
* - `NOT_PROMOTED` (409): the run never went live.
|
|
3111
|
+
* - `ROLLBACK_WINDOW_CLOSED` (409): more than 30 days since it went live.
|
|
3112
|
+
* - `ROLLBACK_SUPERSEDED` (409): a later version went live since; roll that
|
|
3113
|
+
* one back.
|
|
3114
|
+
* - `NOTHING_TO_RESTORE` (409): nothing is recorded as serving before it.
|
|
3115
|
+
* - `RUN_ACTIVE` (409): an attempt is being compared with this version, or is
|
|
3116
|
+
* being built on top of it. Cancel that attempt (`cancelTrainingRun`)
|
|
3117
|
+
* first, or wait for it to finish.
|
|
3118
|
+
* - `ROLLBACK_DELETED_DATA` (409): the version it would put back learned from
|
|
3119
|
+
* a conversation a member deleted since, so it never answers again; a
|
|
3120
|
+
* rebuild without that conversation replaces it.
|
|
3121
|
+
*/
|
|
3122
|
+
export type TrainingRunRollbackRefusalCode = 'NOT_PROMOTED' | 'ROLLBACK_WINDOW_CLOSED' | 'ROLLBACK_SUPERSEDED' | 'NOTHING_TO_RESTORE' | 'RUN_ACTIVE' | 'ROLLBACK_DELETED_DATA';
|
|
3123
|
+
/** A pipeline whose model a trace delete rebuilds: its live version learned from the deleted conversation. */
|
|
3124
|
+
export interface LoopTraceRebuild {
|
|
3125
|
+
pipeline_id: string;
|
|
3126
|
+
pipeline_name: string;
|
|
3127
|
+
}
|
|
3128
|
+
/**
|
|
3129
|
+
* `DELETE /api/loop/traces/{id}`. `rebuilds` names every pipeline whose live
|
|
3130
|
+
* version learned from the conversation: each rebuilds its model from the
|
|
3131
|
+
* base without it. A pipeline that is switched off is named too; its rebuild
|
|
3132
|
+
* starts when it is switched back on. Empty when none did; absent on an
|
|
3133
|
+
* older service.
|
|
3134
|
+
*/
|
|
3135
|
+
export interface LoopTraceDeleteResponse {
|
|
3136
|
+
deleted: boolean;
|
|
3137
|
+
trace_id: string;
|
|
3138
|
+
rebuilds?: LoopTraceRebuild[];
|
|
3139
|
+
}
|
|
2971
3140
|
export interface TrainingRunCancelRequest {
|
|
2972
3141
|
reason?: string;
|
|
2973
3142
|
}
|
|
@@ -3004,6 +3173,13 @@ export interface TrainingRun {
|
|
|
3004
3173
|
attempts: number;
|
|
3005
3174
|
next_poll_at: string | null;
|
|
3006
3175
|
last_reason: string | null;
|
|
3176
|
+
/**
|
|
3177
|
+
* Why it failed or was set aside, as a code. Two belong to a pipeline that
|
|
3178
|
+
* builds on its live version: `CHAIN_MOVED` (it was built on a version that
|
|
3179
|
+
* was rolled back while it trained; it is superseded, no comparison was
|
|
3180
|
+
* paid for, and its samples go to the next attempt) and `NO_MERGED_EXPORT`
|
|
3181
|
+
* (its training finished without the merged model it is served as).
|
|
3182
|
+
*/
|
|
3007
3183
|
error_code: string | null;
|
|
3008
3184
|
last_error: string | null;
|
|
3009
3185
|
/**
|
|
@@ -3030,6 +3206,8 @@ export interface TrainingRun {
|
|
|
3030
3206
|
spent_eval_cents: number;
|
|
3031
3207
|
train_rows: number | null;
|
|
3032
3208
|
holdout_rows: number | null;
|
|
3209
|
+
/** What it was trained on top of, and on which samples. See {@link Lineage}. */
|
|
3210
|
+
lineage: Lineage;
|
|
3033
3211
|
train_dataset_id: string | null;
|
|
3034
3212
|
/** Null on a run made since the held-back rows stopped leaving the loop; set on older runs. */
|
|
3035
3213
|
holdout_dataset_id: string | null;
|
|
@@ -3056,8 +3234,17 @@ export interface TrainingRun {
|
|
|
3056
3234
|
model_family: string | null;
|
|
3057
3235
|
/** Machine time the run held (training job plus comparison machine); null until known. */
|
|
3058
3236
|
gpu_seconds: number | null;
|
|
3059
|
-
/** The trainer's loss on its validation slice, as {@link Evaluation.trainer_eval_loss}. */
|
|
3237
|
+
/** The trainer's loss on its validation slice, as {@link Evaluation.trainer_eval_loss}: at the checkpoint the attempt kept. */
|
|
3060
3238
|
training_eval_loss: number | null;
|
|
3239
|
+
/** Optimizer steps the attempt's training job took, as the trainer counted them. Null until the job has ended. */
|
|
3240
|
+
trained_steps: number | null;
|
|
3241
|
+
/**
|
|
3242
|
+
* One plain sentence about how the platform trained the attempt: the steps
|
|
3243
|
+
* it took, the passes planned over its samples and the checkpoint it kept.
|
|
3244
|
+
* Null on an attempt the step-aware recipe did not train, and until its job
|
|
3245
|
+
* has ended.
|
|
3246
|
+
*/
|
|
3247
|
+
training_summary: string | null;
|
|
3061
3248
|
candidate_deployment_id: string | null;
|
|
3062
3249
|
candidate_name: string | null;
|
|
3063
3250
|
candidate_seq: number;
|
|
@@ -3354,7 +3541,8 @@ export type ReportSort = 'ordinal' | 'score_change';
|
|
|
3354
3541
|
* `candidate_passed: false, incumbent_passed: true` is the regressions,
|
|
3355
3542
|
* `candidate_passed: false` alone every row the new model did not pass.
|
|
3356
3543
|
* `q` (2 to 200 characters) searches, case-insensitively, what a
|
|
3357
|
-
* reader of each row sees -- the conversation
|
|
3544
|
+
* reader of each row sees -- the conversation (not the system prompt every row
|
|
3545
|
+
* shares), the right answer, both answers
|
|
3358
3546
|
* and their tool calls, every reason -- read from storage by the service; the
|
|
3359
3547
|
* answer's `searched` says how many rows it searched. A search can be refused
|
|
3360
3548
|
* 429 `RATE_LIMITED`, with `Retry-After`, while this workspace already has a
|
|
@@ -3577,6 +3765,13 @@ export interface BenchmarkRun {
|
|
|
3577
3765
|
prompt_tokens: number;
|
|
3578
3766
|
completion_tokens: number;
|
|
3579
3767
|
spent_cents: number;
|
|
3768
|
+
/**
|
|
3769
|
+
* How many of the conversations this replay asked the version under test
|
|
3770
|
+
* learned -- trained on itself, or learned by a version it was built on
|
|
3771
|
+
* ("N of these conversations were learned by this version"): a score on
|
|
3772
|
+
* those is partly recall. Null for a version the merge chain did not build.
|
|
3773
|
+
*/
|
|
3774
|
+
learned_by_version: number | null;
|
|
3580
3775
|
status: BenchmarkRunStatus;
|
|
3581
3776
|
/** The whole explanation when there is no score, so never empty on one. */
|
|
3582
3777
|
status_reason: string | null;
|
|
@@ -3609,6 +3804,8 @@ export interface BenchmarkHistoryPoint {
|
|
|
3609
3804
|
incumbent_pass_rate: number | null;
|
|
3610
3805
|
candidate_pass_rate: number | null;
|
|
3611
3806
|
checkpoint_id: string | null;
|
|
3807
|
+
/** The replay's {@link BenchmarkRun.learned_by_version}. */
|
|
3808
|
+
learned_by_version: number | null;
|
|
3612
3809
|
decision: TrainingDecision | null;
|
|
3613
3810
|
status: BenchmarkRunStatus;
|
|
3614
3811
|
status_reason: string | null;
|
|
@@ -3743,7 +3940,10 @@ export interface BenchmarkRetireParams {
|
|
|
3743
3940
|
* it (`paused_reason` says why).
|
|
3744
3941
|
* - `RUN_ACTIVE` (409): one attempt at a time, so two cannot race to promote.
|
|
3745
3942
|
* - `VERSION_LIMIT_REACHED` (409): the pipeline has made `max_versions`
|
|
3746
|
-
* versions. Raise the limit or remove it.
|
|
3943
|
+
* versions and owes no rebuild. Raise the limit or remove it. A rebuild
|
|
3944
|
+
* after a deleted conversation (`chain.rebuild`) is not held by the limit:
|
|
3945
|
+
* Train now starts it, and each one that goes live is a version, so the
|
|
3946
|
+
* pipeline can end past its limit.
|
|
3747
3947
|
* - `MONTHLY_LIMIT_REACHED` (409): the most this month's attempts can have
|
|
3748
3948
|
* cost plus the most one more may cost would pass the pipeline's internal
|
|
3749
3949
|
* monthly limit. The body carries the figures:
|
|
@@ -3909,6 +4109,8 @@ export interface PipelineActiveRun {
|
|
|
3909
4109
|
last_error: string | null;
|
|
3910
4110
|
train_rows: number | null;
|
|
3911
4111
|
holdout_rows: number | null;
|
|
4112
|
+
/** What it trains on top of, and on which samples. See {@link Lineage}. */
|
|
4113
|
+
lineage: Lineage;
|
|
3912
4114
|
/**
|
|
3913
4115
|
* Its comparison scored at least one conversation. False on a reported run
|
|
3914
4116
|
* means the held-back set was too small to compare anything: that run is
|
|
@@ -3943,6 +4145,8 @@ export interface PipelineVersion {
|
|
|
3943
4145
|
checkpoint_id: string | null;
|
|
3944
4146
|
train_rows: number | null;
|
|
3945
4147
|
holdout_rows: number | null;
|
|
4148
|
+
/** What it was trained on top of, and on which samples. See {@link Lineage}. */
|
|
4149
|
+
lineage: Lineage;
|
|
3946
4150
|
/** Head-to-head against the champion of its day, on conversations neither trained on. */
|
|
3947
4151
|
rows_scored: number | null;
|
|
3948
4152
|
win_rate: number | null;
|
|
@@ -4010,10 +4214,12 @@ export interface PipelineVersion {
|
|
|
4010
4214
|
*/
|
|
4011
4215
|
/**
|
|
4012
4216
|
* Why an attempt was made: new data (`data`, a recipe variant included), the
|
|
4013
|
-
* schedule (`schedule`), a member's train-now (`manual`),
|
|
4014
|
-
* model trained
|
|
4217
|
+
* schedule (`schedule`), a member's train-now (`manual`), the challenger
|
|
4218
|
+
* model trained from its own base on all samples (`challenger`), or a rebuild
|
|
4219
|
+
* from the base after a member deleted a conversation the live version
|
|
4220
|
+
* learned from (`rebuild`).
|
|
4015
4221
|
*/
|
|
4016
|
-
export type PipelineAttemptTrigger = 'data' | 'schedule' | 'manual' | 'challenger';
|
|
4222
|
+
export type PipelineAttemptTrigger = 'data' | 'schedule' | 'manual' | 'challenger' | 'rebuild';
|
|
4017
4223
|
/** What became of an attempt. */
|
|
4018
4224
|
export type PipelineAttemptOutcome = 'became_version' | 'not_better' | 'inconclusive' | 'stopped' | 'failed' | 'running' | 'waiting_for_review';
|
|
4019
4225
|
/** One attempt: every training run a pipeline makes, winner or not. */
|
|
@@ -4033,6 +4239,8 @@ export interface PipelineAttempt {
|
|
|
4033
4239
|
/** The samples its set trained on and held back; null until its set was built. */
|
|
4034
4240
|
train_rows: number | null;
|
|
4035
4241
|
holdout_rows: number | null;
|
|
4242
|
+
/** What it was trained on top of, and on which samples. See {@link Lineage}. */
|
|
4243
|
+
lineage: Lineage;
|
|
4036
4244
|
/** The version it became, when it became one. */
|
|
4037
4245
|
version: number | null;
|
|
4038
4246
|
created_at: string;
|
|
@@ -4062,7 +4270,10 @@ export interface PipelineModelRef {
|
|
|
4062
4270
|
/**
|
|
4063
4271
|
* When a new version takes over. `holdout` (the default) when it wins on the
|
|
4064
4272
|
* held-back set; `all`, `primary`, `k_of_n` (with `k`) and `weighted` also
|
|
4065
|
-
* weigh the pipeline's benchmarks; `manual` never on its own.
|
|
4273
|
+
* weigh the pipeline's benchmarks; `manual` never on its own. A rebuild after
|
|
4274
|
+
* a deleted conversation (trigger `rebuild`) is the exception under every
|
|
4275
|
+
* policy but `manual`: it goes live unless it is clearly worse than the live
|
|
4276
|
+
* version. Under `manual` it waits for a member like any attempt.
|
|
4066
4277
|
*/
|
|
4067
4278
|
export interface PipelinePromotion {
|
|
4068
4279
|
policy: 'holdout' | 'all' | 'primary' | 'k_of_n' | 'weighted' | 'manual';
|
|
@@ -4113,8 +4324,8 @@ export interface PipelineHoldout {
|
|
|
4113
4324
|
* `first_version`: `needed` usable samples the first attempt needs -- the
|
|
4114
4325
|
* rows it trains on plus the ones held back to judge it. `new_rows`: `needed`
|
|
4115
4326
|
* usable samples the last trained attempt did not have, which is
|
|
4116
|
-
* `floors.new_samples`
|
|
4117
|
-
*
|
|
4327
|
+
* `floors.new_samples` (1,000), however many rows that attempt trained on.
|
|
4328
|
+
* `have`, `short` and `message` are
|
|
4118
4329
|
* measured on the detail only. There, after an attempt has trained, `short`
|
|
4119
4330
|
* is how many more new samples have to arrive: new samples fill the held-back
|
|
4120
4331
|
* set first, so after the held-back share is raised it can be more than
|
|
@@ -4146,6 +4357,34 @@ export interface PipelineLiveDeployment {
|
|
|
4146
4357
|
status: PipelineLiveDeploymentStatus;
|
|
4147
4358
|
checked_at: string;
|
|
4148
4359
|
}
|
|
4360
|
+
/**
|
|
4361
|
+
* A rebuild on its way: a member deleted `conversations` conversations the
|
|
4362
|
+
* live version learned from, first at `since`. The next attempt trains from
|
|
4363
|
+
* the base on every remaining sample without waiting for new ones; until it
|
|
4364
|
+
* goes live, the live version still holds what it learned from them.
|
|
4365
|
+
*/
|
|
4366
|
+
export interface PipelineChainRebuild {
|
|
4367
|
+
reason: 'conversation_deleted';
|
|
4368
|
+
conversations: number;
|
|
4369
|
+
since: string;
|
|
4370
|
+
}
|
|
4371
|
+
/**
|
|
4372
|
+
* The live version as what it is made of: the pinned base with `versions`
|
|
4373
|
+
* merged into it in that order, the live one last. Null when nothing this
|
|
4374
|
+
* pipeline trained is live on its current model, and when the pipeline does
|
|
4375
|
+
* not build on its live version (then every attempt trains from the base on
|
|
4376
|
+
* all samples). `learned_samples` is how many samples those versions learned
|
|
4377
|
+
* (the next attempt does not train them again, except a replayed fifth);
|
|
4378
|
+
* `not_learned_samples` is how many were trained by attempts that did not go
|
|
4379
|
+
* live, so the next attempt trains them again. Null on the list.
|
|
4380
|
+
*/
|
|
4381
|
+
export interface PipelineChain {
|
|
4382
|
+
versions: number[];
|
|
4383
|
+
live_version: number;
|
|
4384
|
+
learned_samples: number;
|
|
4385
|
+
not_learned_samples: number | null;
|
|
4386
|
+
rebuild: PipelineChainRebuild | null;
|
|
4387
|
+
}
|
|
4149
4388
|
export interface Pipeline {
|
|
4150
4389
|
rule_id: string;
|
|
4151
4390
|
rule_name: string;
|
|
@@ -4163,8 +4402,9 @@ export interface Pipeline {
|
|
|
4163
4402
|
/** A conversation carrying ANY of these tags is this pipeline's data. */
|
|
4164
4403
|
tags: string[];
|
|
4165
4404
|
/**
|
|
4166
|
-
* A second base model trained on
|
|
4167
|
-
*
|
|
4405
|
+
* A second base model, trained from its own base on all samples after each
|
|
4406
|
+
* attempt (it never builds on this pipeline's versions, which are another
|
|
4407
|
+
* model); it becomes the next version only if it beats the live one, and then it
|
|
4168
4408
|
* becomes the pipeline's model (`base_model_id`) while the model it replaced
|
|
4169
4409
|
* becomes this one. `revision` is the commit the server pinned.
|
|
4170
4410
|
*/
|
|
@@ -4175,7 +4415,11 @@ export interface Pipeline {
|
|
|
4175
4415
|
/** Rows the next version holds back and never trains on. Detail only; null on the list. */
|
|
4176
4416
|
holdout_size: number | null;
|
|
4177
4417
|
minimums: PipelineMinimums;
|
|
4178
|
-
/**
|
|
4418
|
+
/**
|
|
4419
|
+
* The pinned base every version is trained over: from it, or, where the
|
|
4420
|
+
* pipeline builds on its live version (`chain`), on top of the versions
|
|
4421
|
+
* before it merged into it.
|
|
4422
|
+
*/
|
|
4179
4423
|
base_model_id: string;
|
|
4180
4424
|
base_model_revision: string;
|
|
4181
4425
|
/** The version serving now; null when what serves is not a version of this pipeline. */
|
|
@@ -4188,6 +4432,8 @@ export interface Pipeline {
|
|
|
4188
4432
|
* serves is not a deployment, or before the first check of that deployment.
|
|
4189
4433
|
*/
|
|
4190
4434
|
live_deployment: PipelineLiveDeployment | null;
|
|
4435
|
+
/** The live version as the versions merged into it. See {@link PipelineChain}. */
|
|
4436
|
+
chain: PipelineChain | null;
|
|
4191
4437
|
/** What the pipeline's history takes up in storage. */
|
|
4192
4438
|
storage: LoopStorage;
|
|
4193
4439
|
/** The name the app calls. It does not change when a version is promoted. */
|
|
@@ -4503,13 +4749,11 @@ export interface PipelineFloors {
|
|
|
4503
4749
|
* back more needs more; its own `minimums` says how many).
|
|
4504
4750
|
*/
|
|
4505
4751
|
first_version_samples: number;
|
|
4506
|
-
/** The fewest new samples any later attempt waits for. */
|
|
4507
|
-
new_samples: number;
|
|
4508
4752
|
/**
|
|
4509
|
-
* The
|
|
4510
|
-
*
|
|
4753
|
+
* The new samples every later attempt waits for, counted since the last
|
|
4754
|
+
* trained attempt, however many rows that attempt trained on.
|
|
4511
4755
|
*/
|
|
4512
|
-
|
|
4756
|
+
new_samples: number;
|
|
4513
4757
|
/** The held-back rule a pipeline chooses its share within. */
|
|
4514
4758
|
held_back: HeldBackFloors;
|
|
4515
4759
|
}
|
|
@@ -4603,6 +4847,8 @@ export interface VersionBenchmarkScore {
|
|
|
4603
4847
|
rows_scored: number | null;
|
|
4604
4848
|
rows_total: number | null;
|
|
4605
4849
|
items_digest: string | null;
|
|
4850
|
+
/** The replay's {@link BenchmarkRun.learned_by_version}; null when not measured. */
|
|
4851
|
+
learned_by_version: number | null;
|
|
4606
4852
|
comparable: boolean;
|
|
4607
4853
|
why: string | null;
|
|
4608
4854
|
/** The share each side passed (score 0.70 or more); null when there is no score. */
|
|
@@ -4725,6 +4971,8 @@ export interface AttemptCompareSide {
|
|
|
4725
4971
|
decision: TrainingDecision | null;
|
|
4726
4972
|
train_rows: number | null;
|
|
4727
4973
|
holdout_rows: number | null;
|
|
4974
|
+
/** What it was trained on top of, and on which samples. See {@link Lineage}. */
|
|
4975
|
+
lineage: Lineage;
|
|
4728
4976
|
finished_at: string | null;
|
|
4729
4977
|
holdout: AttemptHoldout;
|
|
4730
4978
|
}
|