runbios-sdk 0.2.21-dev.300 → 0.2.21-dev.304
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/resources/loop.d.ts +36 -10
- package/dist/resources/loop.js +35 -6
- package/dist/resources/training.js +18 -0
- package/dist/types.d.ts +259 -21
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -36,7 +36,7 @@ import { Loop } from './resources/loop.js';
|
|
|
36
36
|
* SDK version. Sent as part of the User-Agent header.
|
|
37
37
|
* Must match package.json "version" -- enforced by a contract test.
|
|
38
38
|
*/
|
|
39
|
-
export declare const VERSION = "0.2.21-dev.
|
|
39
|
+
export declare const VERSION = "0.2.21-dev.304";
|
|
40
40
|
export declare class RunBiOS {
|
|
41
41
|
/** Search models, fetch configs, check adapter compatibility. */
|
|
42
42
|
readonly models: Models;
|
|
@@ -75,7 +75,7 @@ export { Training } from './resources/training.js';
|
|
|
75
75
|
export { Wallet } from './resources/wallet.js';
|
|
76
76
|
export { GPU, type GPURecommendation } from './resources/gpu.js';
|
|
77
77
|
export { Inference, validateChatRequest, parseSSE, buildInferenceRequest, contextSizingBasis, type ContextSizingBasis, CONTEXT_DEFAULT_CEILING, CONTEXT_EDITABLE_FLOOR, type ChatMessage, type FunctionTool, type ChatCompletionParams, type ChatCompletionResponse, type ChatCompletionChunk, type InferenceVerbParams, type CompletionParams, type EmbeddingParams, type RerankParams, type AnthropicMessageParams, type ServerlessUsageWindow, type ServerlessTimeseriesMetric, type ServerlessWorkspaceLimits, type ServerlessUsageEnvelope, type ServerlessUsageOverviewResponse, type ServerlessUsageRowsResponse, type ServerlessUsageTimeseriesResponse, type ServerlessUsageDailyResponse, type ServerlessSavingsTotals, type ServerlessSavingsPeriod, type ServerlessUsageSavingsResponse, } from './resources/inference.js';
|
|
78
|
-
export type { BiOSConfig, PaginatedResponse, ApiErrorBody, AvailableGpuAlternative, CapacityMinimumRequirement, InferenceBookingAccepted, Model, ModelSearchParams, ModelSearchResponse, ModelDetailResponse, InferenceModel, InferenceModelListResponse, ModelConfig, Dataset, DatasetListParams, DatasetListResponse, DatasetUploadParams, DatasetPreview, DatasetPreviewParams, DatasetImportHFParams, DatasetRegisterHFParams, DatasetHubSearchParams, DatasetHubPreviewParams, DatasetValidation, DatasetFormatVariant, DatasetFormatSpec, DatasetFormatSpecs, DatasetStorageUsage, TrainingMethod, RLHFAlgorithm, AdapterType, TrainingJobStatus, TrainingStatusFilter, TrainingCreateParams, TrainingListParams, TrainingListResponse, TrainingJob, TrainingMetrics, TrainingResourceMetrics, TrainingDeviceMetrics, MetricPoint, MetricGraphConfig, TrainingCheckpoint, TrainingLogs, TrainingLogEntry, TrainingStopResponse, TrainingResumeResponse, CanonicalTrainingRequest, TrainingPreflightDataset, TrainingPreflightWarning, TrainingPreflightResponse, TrainingCapabilityChoice, TrainingConfigFieldCapability, TrainingCapabilities, GPUChoice, WalletBalance, Transaction, TransactionListResponse, TransactionListParams, GPUInfo, GPUPricingResponse, GPUOptionsParams, TrainingRecommendParams, TrainingAdvisorVerdict, TrainingAdvisorRecommendation, TrainingAdvisorUserShape, TrainingAdvisorJustification, TrainingAdvisorBasis, GPUOption, GPUOptionSuggestion, GPUOptionsResponse, InferenceStatus, InferenceCreateParams, InferenceUpdateParams, InferenceDeployment, InferenceDeploymentSummary, InferenceCreateResponse, InferenceListResponse, InferenceUpdateResponse, InferenceLifecycleResponse, InferenceDeleteResponse, InferenceGPUOptionMarket, InferenceGPUOption, InferenceGPUAlternative, InferenceGPUOptionsParams, InferenceGPUOptionsResponse, InferenceCanonicalRequest, InferencePreflightWarning, InferencePreflightResponse, AdapterCompatibility, AdapterCompatibilityResponse, AdapterCompatibilityParams, SupportedArchitecture, ArchitectureScope, SupportedArchitecturesResponse, ApiKey, ApiKeyScope, ApiKeyIntrospection, Organization, OrgMember, OrgInvite, Workspace, Integration, IntegrationCreateParams, StorageUsage, StorageObject, PromptTokensDetails, ChatCompletionUsage, ServingKind, TrainingRulePausedReason, TrainingTrigger, TrainingRecipe, TrainingRunState, TrainingVerdict, TrainingDecision, EvaluationStatus, EvaluationItemStatus, ItemErrorCode, EvaluationWinner, EvaluationItemWinnerFilter, ReportStatusFilter, ReportSort, EvaluationWarningCode, TrainingToolCall, TrainingMessage, TrainingRuleServing, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, TrainingRunListParams, TrainingRun, TrainingRunSummary, TrainingRunEvent, TrainingRunLinks, TrainingRunActions, EvaluationRef, EvaluationDecoding, EvaluationJudgeDimension, EvaluationDimensionScore, EvaluationWarning, EvaluationMargin, Evaluation, EvaluationItemJudge, EvaluationReferenceKind, EvaluationReference, EvaluationPairwiseOrder, EvaluationItemPairwise, EvaluationItemSide, EvaluationItem, EvaluationItemListParams, BenchmarkStatus, BenchmarkSourceKind, BenchmarkRunStatus, BenchmarkScoring, BenchmarkCheck, BenchmarkDimensionScore, Benchmark, BenchmarkItem, BenchmarkRun, BenchmarkRunItemSide, BenchmarkRunItem, BenchmarkRunItemListParams, BenchmarkHistoryPoint, BenchmarkSource, BenchmarkCreateParams, BenchmarkListParams, BenchmarkItemListParams, BenchmarkHistoryParams, BenchmarkRetireParams, InferenceAlias, InferenceAliasRequest, TrainingRuleDeleteResponse, TrainingRunListResponse, TrainingRunBenchmarkRun, TrainingRunResponse, TrainingRunActionResponse, EvaluationItemsResponse, BenchmarkRunItemsResponse, InferenceAliasListResponse, InferenceAliasResponse, InferenceAliasDeleteResponse, BenchmarkListResponse, BenchmarkItemsResponse, BenchmarkHistoryResponse, TrainingRuleRunRefusalCode, TrainingRuleSaveRefusalCode, TrainingRuleMonthlyLimitRefusal, PipelineStatus, PipelineActiveRun, PipelineVersion, PipelineAttempt, PipelineAttemptTrigger, PipelineAttemptOutcome, PipelineLiveDeploymentStatus, PipelineLiveDeployment, LoopStorageWhere, LoopStorage, Pipeline, PipelineListParams, PipelineListResponse, PipelineResponse, PipelineTrainWhen, PipelineSchedule, PipelineWriteRequest, PipelineMutationResponse, PipelineImportParams, PipelineFloors, HeldBackFloors, LoopModel, LoopModelsResponse, LoopTracePipeline, LoopTracePipelinesResponse, LoopMetricsHistoryParams, LoopMetricPoint, LoopSignalDay, LoopMetricsHistory, PromotionPolicyState, RuleBenchmark, PipelineBenchmarksResponse, PipelineBenchmarkAttachParams, PipelineBenchmarkMoveParams, CompareBenchmark, VersionBenchmarkScore, PromotionMeasurement, PromotionOutcome, VersionHoldout, VersionScores, VersionScoreDifference, VersionCompareResponse, VersionCompareParams, TrainingRunPromotion, VerdictCheck, VerdictCheckKey, VerdictChecksWithheld, TrainingRunCost, AttemptDirection, AttemptHoldout, AttemptCompareSide, AttemptSharedSample, AttemptSharedFigures, AttemptBenchmarkSide, AttemptCompareBenchmark, AttemptCompareItemSide, AttemptCompareItem, AttemptCompareResponse, AttemptCompareParams, } from './types.js';
|
|
78
|
+
export type { BiOSConfig, PaginatedResponse, ApiErrorBody, AvailableGpuAlternative, CapacityMinimumRequirement, InferenceBookingAccepted, Model, ModelSearchParams, ModelSearchResponse, ModelDetailResponse, InferenceModel, InferenceModelListResponse, ModelConfig, Dataset, DatasetListParams, DatasetListResponse, DatasetUploadParams, DatasetPreview, DatasetPreviewParams, DatasetImportHFParams, DatasetRegisterHFParams, DatasetHubSearchParams, DatasetHubPreviewParams, DatasetValidation, DatasetFormatVariant, DatasetFormatSpec, DatasetFormatSpecs, DatasetStorageUsage, TrainingMethod, RLHFAlgorithm, AdapterType, TrainingJobStatus, TrainingStatusFilter, TrainingCreateParams, TrainingStartFromParams, TrainingStartFrom, TrainingListParams, TrainingListResponse, TrainingJob, TrainingMetrics, TrainingResourceMetrics, TrainingDeviceMetrics, MetricPoint, MetricGraphConfig, TrainingCheckpoint, TrainingLogs, TrainingLogEntry, TrainingStopResponse, TrainingResumeResponse, CanonicalTrainingRequest, TrainingPreflightDataset, TrainingPreflightWarning, TrainingPreflightResponse, TrainingCapabilityChoice, TrainingConfigFieldCapability, TrainingCapabilities, TrainingFeature, GPUChoice, WalletBalance, Transaction, TransactionListResponse, TransactionListParams, GPUInfo, GPUPricingResponse, GPUOptionsParams, TrainingRecommendParams, TrainingAdvisorVerdict, TrainingAdvisorRecommendation, TrainingAdvisorUserShape, TrainingAdvisorJustification, TrainingAdvisorBasis, GPUOption, GPUOptionSuggestion, GPUOptionsResponse, InferenceStatus, InferenceCreateParams, InferenceUpdateParams, InferenceDeployment, InferenceDeploymentSummary, InferenceCreateResponse, InferenceListResponse, InferenceUpdateResponse, InferenceLifecycleResponse, InferenceDeleteResponse, InferenceGPUOptionMarket, InferenceGPUOption, InferenceGPUAlternative, InferenceGPUOptionsParams, InferenceGPUOptionsResponse, InferenceCanonicalRequest, InferencePreflightWarning, InferencePreflightResponse, AdapterCompatibility, AdapterCompatibilityResponse, AdapterCompatibilityParams, SupportedArchitecture, ArchitectureScope, SupportedArchitecturesResponse, ApiKey, ApiKeyScope, ApiKeyIntrospection, Organization, OrgMember, OrgInvite, Workspace, Integration, IntegrationCreateParams, StorageUsage, StorageObject, PromptTokensDetails, ChatCompletionUsage, ServingKind, TrainingRulePausedReason, TrainingTrigger, TrainingRecipe, TrainingRunState, TrainingVerdict, TrainingDecision, EvaluationStatus, EvaluationItemStatus, ItemErrorCode, EvaluationWinner, EvaluationItemWinnerFilter, ReportStatusFilter, ReportSort, EvaluationWarningCode, TrainingToolCall, TrainingMessage, TrainingRuleServing, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, TrainingRunListParams, TrainingRun, TrainingRunSummary, TrainingRunEvent, TrainingRunLinks, TrainingRunActions, EvaluationRef, EvaluationDecoding, EvaluationJudgeDimension, EvaluationDimensionScore, EvaluationWarning, EvaluationMargin, Evaluation, EvaluationItemJudge, EvaluationReferenceKind, EvaluationReference, EvaluationPairwiseOrder, EvaluationItemPairwise, EvaluationItemSide, EvaluationItem, EvaluationItemListParams, BenchmarkStatus, BenchmarkSourceKind, BenchmarkRunStatus, BenchmarkScoring, BenchmarkCheck, BenchmarkDimensionScore, Benchmark, BenchmarkItem, BenchmarkRun, BenchmarkRunItemSide, BenchmarkRunItem, BenchmarkRunItemListParams, BenchmarkHistoryPoint, BenchmarkSource, BenchmarkCreateParams, BenchmarkListParams, BenchmarkItemListParams, BenchmarkHistoryParams, BenchmarkRetireParams, InferenceAlias, InferenceAliasRequest, TrainingRuleDeleteResponse, TrainingRunListResponse, TrainingRunBenchmarkRun, TrainingRunResponse, TrainingRunActionResponse, EvaluationItemsResponse, BenchmarkRunItemsResponse, InferenceAliasListResponse, InferenceAliasResponse, InferenceAliasDeleteResponse, BenchmarkListResponse, BenchmarkItemsResponse, BenchmarkHistoryResponse, TrainingRuleRunRefusalCode, TrainingRuleSaveRefusalCode, TrainingRuleMonthlyLimitRefusal, PipelineStatus, PipelineActiveRun, PipelineVersion, PipelineAttempt, PipelineAttemptTrigger, Lineage, PipelineChain, PipelineChainRebuild, TrainingRunRollbackRefusalCode, LoopTraceRebuild, LoopTraceDeleteResponse, PipelineAttemptOutcome, PipelineLiveDeploymentStatus, PipelineLiveDeployment, LoopStorageWhere, LoopStorage, Pipeline, PipelineListParams, PipelineListResponse, PipelineResponse, PipelineTrainWhen, PipelineSchedule, PipelineWriteRequest, PipelineMutationResponse, PipelineImportParams, PipelineFloors, HeldBackFloors, LoopModel, LoopModelsResponse, LoopTracePipeline, LoopTracePipelinesResponse, LoopMetricsHistoryParams, LoopMetricPoint, LoopSignalDay, LoopMetricsHistory, PromotionPolicyState, RuleBenchmark, PipelineBenchmarksResponse, PipelineBenchmarkAttachParams, PipelineBenchmarkMoveParams, CompareBenchmark, VersionBenchmarkScore, PromotionMeasurement, PromotionOutcome, VersionHoldout, VersionScores, VersionScoreDifference, VersionCompareResponse, VersionCompareParams, TrainingRunPromotion, VerdictCheck, VerdictCheckKey, VerdictChecksWithheld, TrainingRunCost, AttemptDirection, AttemptHoldout, AttemptCompareSide, AttemptSharedSample, AttemptSharedFigures, AttemptBenchmarkSide, AttemptCompareBenchmark, AttemptCompareItemSide, AttemptCompareItem, AttemptCompareResponse, AttemptCompareParams, } from './types.js';
|
|
79
79
|
/** The closed set of run states a training run never leaves. */
|
|
80
80
|
export { TERMINAL_RUN_STATES } from './types.js';
|
|
81
81
|
/** The most versions a pipeline may be set to make. */
|
package/dist/index.js
CHANGED
|
@@ -36,7 +36,7 @@ import { Loop } from './resources/loop.js';
|
|
|
36
36
|
* SDK version. Sent as part of the User-Agent header.
|
|
37
37
|
* Must match package.json "version" -- enforced by a contract test.
|
|
38
38
|
*/
|
|
39
|
-
export const VERSION = '0.2.21-dev.
|
|
39
|
+
export const VERSION = '0.2.21-dev.304';
|
|
40
40
|
export class RunBiOS {
|
|
41
41
|
/** Search models, fetch configs, check adapter compatibility. */
|
|
42
42
|
models;
|
package/dist/resources/loop.d.ts
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
import type { HttpClient } from '../client.js';
|
|
2
|
-
import type { LoopTrace, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopImportParams, LoopImportResult, LoopSignal, LoopSignalParams, LoopLabel, LoopLabelCount, LoopConfig, LoopConfigParams, LoopStats, LoopTracePipelinesResponse, LoopMetricsHistory, LoopMetricsHistoryParams, TrainingRuleDeleteResponse, TrainingRunListParams, TrainingRunListResponse, TrainingRunResponse, TrainingRunActionResponse, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, Evaluation, EvaluationItemListParams, EvaluationItemsResponse, Benchmark, BenchmarkCreateParams, BenchmarkHistoryParams, BenchmarkHistoryResponse, BenchmarkItemListParams, BenchmarkItemsResponse, BenchmarkListParams, BenchmarkListResponse, BenchmarkRetireParams, BenchmarkRun, BenchmarkRunItemListParams, BenchmarkRunItemsResponse, Pipeline, PipelineListParams, PipelineListResponse, PipelineWriteRequest, PipelineMutationResponse, LoopModelsResponse, PipelineImportParams, PipelineBenchmarksResponse, PipelineBenchmarkAttachParams, PipelineBenchmarkMoveParams, VersionCompareResponse, VersionCompareParams, AttemptCompareResponse, AttemptCompareParams } from '../types.js';
|
|
2
|
+
import type { LoopTrace, LoopTraceDeleteResponse, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopImportParams, LoopImportResult, LoopSignal, LoopSignalParams, LoopLabel, LoopLabelCount, LoopConfig, LoopConfigParams, LoopStats, LoopTracePipelinesResponse, LoopMetricsHistory, LoopMetricsHistoryParams, TrainingRuleDeleteResponse, TrainingRunListParams, TrainingRunListResponse, TrainingRunResponse, TrainingRunActionResponse, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, Evaluation, EvaluationItemListParams, EvaluationItemsResponse, Benchmark, BenchmarkCreateParams, BenchmarkHistoryParams, BenchmarkHistoryResponse, BenchmarkItemListParams, BenchmarkItemsResponse, BenchmarkListParams, BenchmarkListResponse, BenchmarkRetireParams, BenchmarkRun, BenchmarkRunItemListParams, BenchmarkRunItemsResponse, Pipeline, PipelineListParams, PipelineListResponse, PipelineWriteRequest, PipelineMutationResponse, LoopModelsResponse, PipelineImportParams, PipelineBenchmarksResponse, PipelineBenchmarkAttachParams, PipelineBenchmarkMoveParams, VersionCompareResponse, VersionCompareParams, AttemptCompareResponse, AttemptCompareParams } from '../types.js';
|
|
3
3
|
/**
|
|
4
4
|
* The Conscious Loop -- pipelines that keep improving a model from the
|
|
5
5
|
* conversations you send them.
|
|
6
6
|
*
|
|
7
7
|
* A pipeline is one use case: a model, the tags whose samples train it, when
|
|
8
8
|
* to train, and how a new version is promoted. Every time it trains it makes
|
|
9
|
-
* an attempt; an attempt that
|
|
9
|
+
* an attempt; an attempt that goes live becomes the next version: one that
|
|
10
|
+
* beats the live model, one a member approves, or, after a deleted
|
|
11
|
+
* conversation, a rebuild without it that is not clearly worse (a `manual`
|
|
12
|
+
* pipeline waits for a member even then).
|
|
10
13
|
*
|
|
11
14
|
* Nothing is captured until a source is switched on, and you can switch it
|
|
12
15
|
* off again at any time. Creating a pipeline switches on the source named by
|
|
@@ -152,13 +155,16 @@ export declare class Loop {
|
|
|
152
155
|
*
|
|
153
156
|
* It also leaves every training set built from it, including sets already
|
|
154
157
|
* exported -- so a customer asking you to delete a conversation genuinely
|
|
155
|
-
* removes it from what the model will learn next.
|
|
156
|
-
*
|
|
158
|
+
* removes it from what the model will learn next. Where the live version of
|
|
159
|
+
* a pipeline learned from it, that pipeline rebuilds its model from the
|
|
160
|
+
* base on every remaining sample, without it: `rebuilds` names each one.
|
|
161
|
+
* Until the rebuild goes live, the live version still holds what it
|
|
162
|
+
* learned; once it does, no version that learned the conversation can be
|
|
163
|
+
* rolled back to, and their stored weights are deleted. Other finished runs
|
|
164
|
+
* are not affected, and a conversation that expires under the retention
|
|
165
|
+
* setting rebuilds nothing.
|
|
157
166
|
*/
|
|
158
|
-
deleteTrace(id: string): Promise<
|
|
159
|
-
deleted: boolean;
|
|
160
|
-
trace_id: string;
|
|
161
|
-
}>;
|
|
167
|
+
deleteTrace(id: string): Promise<LoopTraceDeleteResponse>;
|
|
162
168
|
/**
|
|
163
169
|
* Record a verdict on a conversation.
|
|
164
170
|
*
|
|
@@ -334,6 +340,14 @@ export declare class Loop {
|
|
|
334
340
|
* The reply carries `serving`: this is the one action besides promotion that
|
|
335
341
|
* changes what answers your traffic, so what it was put back to is returned
|
|
336
342
|
* rather than left to be looked up.
|
|
343
|
+
*
|
|
344
|
+
* Where the pipeline builds on its live version, the version put back is
|
|
345
|
+
* what the next attempt builds on, and it trains again on what the
|
|
346
|
+
* rolled-back version learned. Refusals are a
|
|
347
|
+
* `TrainingRunRollbackRefusalCode`: `RUN_ACTIVE` also while an attempt is
|
|
348
|
+
* being built on this version (cancel it with {@link cancelTrainingRun}
|
|
349
|
+
* first), and `ROLLBACK_DELETED_DATA` when the version it would put back
|
|
350
|
+
* learned from a conversation that was deleted since.
|
|
337
351
|
*/
|
|
338
352
|
rollbackTrainingRun(id: string, params?: TrainingRunRollbackRequest): Promise<TrainingRunActionResponse>;
|
|
339
353
|
/**
|
|
@@ -370,11 +384,16 @@ export declare class Loop {
|
|
|
370
384
|
* The one list {@link createPipeline} accepts a `model.id` from, with the
|
|
371
385
|
* fixed `floors` a pipeline is held to: the samples its first attempt
|
|
372
386
|
* needs, and the new samples each later attempt waits for
|
|
373
|
-
* (`new_samples`,
|
|
374
|
-
* attempt trained on
|
|
387
|
+
* (`new_samples`, counted since the last trained attempt, however many
|
|
388
|
+
* rows that attempt trained on), and the held-back rule
|
|
375
389
|
* (`held_back`: at least `min_rows`, or the share a pipeline chooses from
|
|
376
390
|
* `default_percent` to `max_percent` if that is more, and each attempt is
|
|
377
391
|
* judged on at most `scored_rows` of them). They are read-only.
|
|
392
|
+
* Where a pipeline builds on its live version (its `chain` is set), each
|
|
393
|
+
* attempt trains a new adapter on top of the live version, on what it has
|
|
394
|
+
* not learned yet plus a replay of what it already learned (about a fifth
|
|
395
|
+
* of what it trains on, picked at random, so it is not forgotten), and a
|
|
396
|
+
* winner is merged in and served as one model.
|
|
378
397
|
* When the list cannot be read it throws `ApiError` 503
|
|
379
398
|
* `MODELS_UNAVAILABLE`, whose `body.floors` still carries the floors.
|
|
380
399
|
*/
|
|
@@ -386,6 +405,11 @@ export declare class Loop {
|
|
|
386
405
|
* whenever its data has grown enough (and at its schedule, when it has one),
|
|
387
406
|
* the platform trains a new version on GPUs it chooses, compares it with what
|
|
388
407
|
* serves your traffic, and promotes it by the pipeline's promotion policy.
|
|
408
|
+
* Where the pipeline builds on its live version (`chain`), each attempt
|
|
409
|
+
* waits for 1,000 new samples and trains a new adapter on top of the live
|
|
410
|
+
* version, on what it has not learned plus a replay of what it already
|
|
411
|
+
* learned (about a fifth of what it trains on, picked at random); a winner
|
|
412
|
+
* is merged in.
|
|
389
413
|
* What is charged is the GPU time the runs actually use; the platform's own
|
|
390
414
|
* per-run and monthly limits stop a run that goes wrong. `holdout.percent`
|
|
391
415
|
* (5 to 20, default 5) is the share of its samples held back to judge each
|
|
@@ -423,6 +447,8 @@ export declare class Loop {
|
|
|
423
447
|
* `TrainingRuleRunRefusalCode`. When the attempt in flight is waiting for
|
|
424
448
|
* GPUs to compare (`active_run.state` `waiting_gpus`), this books its two
|
|
425
449
|
* comparison machines again at once instead of starting another attempt.
|
|
450
|
+
* With a rebuild on its way (`chain.rebuild`), it starts the rebuild: from
|
|
451
|
+
* the base on every remaining sample, with no wait for new ones.
|
|
426
452
|
*/
|
|
427
453
|
runPipeline(id: string): Promise<PipelineMutationResponse>;
|
|
428
454
|
/**
|
package/dist/resources/loop.js
CHANGED
|
@@ -4,7 +4,10 @@
|
|
|
4
4
|
*
|
|
5
5
|
* A pipeline is one use case: a model, the tags whose samples train it, when
|
|
6
6
|
* to train, and how a new version is promoted. Every time it trains it makes
|
|
7
|
-
* an attempt; an attempt that
|
|
7
|
+
* an attempt; an attempt that goes live becomes the next version: one that
|
|
8
|
+
* beats the live model, one a member approves, or, after a deleted
|
|
9
|
+
* conversation, a rebuild without it that is not clearly worse (a `manual`
|
|
10
|
+
* pipeline waits for a member even then).
|
|
8
11
|
*
|
|
9
12
|
* Nothing is captured until a source is switched on, and you can switch it
|
|
10
13
|
* off again at any time. Creating a pipeline switches on the source named by
|
|
@@ -199,8 +202,14 @@ export class Loop {
|
|
|
199
202
|
*
|
|
200
203
|
* It also leaves every training set built from it, including sets already
|
|
201
204
|
* exported -- so a customer asking you to delete a conversation genuinely
|
|
202
|
-
* removes it from what the model will learn next.
|
|
203
|
-
*
|
|
205
|
+
* removes it from what the model will learn next. Where the live version of
|
|
206
|
+
* a pipeline learned from it, that pipeline rebuilds its model from the
|
|
207
|
+
* base on every remaining sample, without it: `rebuilds` names each one.
|
|
208
|
+
* Until the rebuild goes live, the live version still holds what it
|
|
209
|
+
* learned; once it does, no version that learned the conversation can be
|
|
210
|
+
* rolled back to, and their stored weights are deleted. Other finished runs
|
|
211
|
+
* are not affected, and a conversation that expires under the retention
|
|
212
|
+
* setting rebuilds nothing.
|
|
204
213
|
*/
|
|
205
214
|
async deleteTrace(id) {
|
|
206
215
|
return this._http.fetchDelete(`/api/loop/traces/${encodeURIComponent(id)}`);
|
|
@@ -401,7 +410,7 @@ export class Loop {
|
|
|
401
410
|
// ── attempts ──────────────────────────────────────────────────────────
|
|
402
411
|
//
|
|
403
412
|
// Every time a pipeline trains it makes an attempt (a training run). The
|
|
404
|
-
// attempts that
|
|
413
|
+
// attempts that went live became its versions.
|
|
405
414
|
/** Attempts, newest first. Filter by pipeline (`rule_id`) or by the state they are sitting in. */
|
|
406
415
|
async listTrainingRuns(params = {}) {
|
|
407
416
|
const q = new URLSearchParams();
|
|
@@ -451,6 +460,14 @@ export class Loop {
|
|
|
451
460
|
* The reply carries `serving`: this is the one action besides promotion that
|
|
452
461
|
* changes what answers your traffic, so what it was put back to is returned
|
|
453
462
|
* rather than left to be looked up.
|
|
463
|
+
*
|
|
464
|
+
* Where the pipeline builds on its live version, the version put back is
|
|
465
|
+
* what the next attempt builds on, and it trains again on what the
|
|
466
|
+
* rolled-back version learned. Refusals are a
|
|
467
|
+
* `TrainingRunRollbackRefusalCode`: `RUN_ACTIVE` also while an attempt is
|
|
468
|
+
* being built on this version (cancel it with {@link cancelTrainingRun}
|
|
469
|
+
* first), and `ROLLBACK_DELETED_DATA` when the version it would put back
|
|
470
|
+
* learned from a conversation that was deleted since.
|
|
454
471
|
*/
|
|
455
472
|
async rollbackTrainingRun(id, params = {}) {
|
|
456
473
|
return this._http.fetchPost(`/api/loop/training-runs/${encodeURIComponent(id)}/rollback`, params);
|
|
@@ -514,11 +531,16 @@ export class Loop {
|
|
|
514
531
|
* The one list {@link createPipeline} accepts a `model.id` from, with the
|
|
515
532
|
* fixed `floors` a pipeline is held to: the samples its first attempt
|
|
516
533
|
* needs, and the new samples each later attempt waits for
|
|
517
|
-
* (`new_samples`,
|
|
518
|
-
* attempt trained on
|
|
534
|
+
* (`new_samples`, counted since the last trained attempt, however many
|
|
535
|
+
* rows that attempt trained on), and the held-back rule
|
|
519
536
|
* (`held_back`: at least `min_rows`, or the share a pipeline chooses from
|
|
520
537
|
* `default_percent` to `max_percent` if that is more, and each attempt is
|
|
521
538
|
* judged on at most `scored_rows` of them). They are read-only.
|
|
539
|
+
* Where a pipeline builds on its live version (its `chain` is set), each
|
|
540
|
+
* attempt trains a new adapter on top of the live version, on what it has
|
|
541
|
+
* not learned yet plus a replay of what it already learned (about a fifth
|
|
542
|
+
* of what it trains on, picked at random, so it is not forgotten), and a
|
|
543
|
+
* winner is merged in and served as one model.
|
|
522
544
|
* When the list cannot be read it throws `ApiError` 503
|
|
523
545
|
* `MODELS_UNAVAILABLE`, whose `body.floors` still carries the floors.
|
|
524
546
|
*/
|
|
@@ -533,6 +555,11 @@ export class Loop {
|
|
|
533
555
|
* whenever its data has grown enough (and at its schedule, when it has one),
|
|
534
556
|
* the platform trains a new version on GPUs it chooses, compares it with what
|
|
535
557
|
* serves your traffic, and promotes it by the pipeline's promotion policy.
|
|
558
|
+
* Where the pipeline builds on its live version (`chain`), each attempt
|
|
559
|
+
* waits for 1,000 new samples and trains a new adapter on top of the live
|
|
560
|
+
* version, on what it has not learned plus a replay of what it already
|
|
561
|
+
* learned (about a fifth of what it trains on, picked at random); a winner
|
|
562
|
+
* is merged in.
|
|
536
563
|
* What is charged is the GPU time the runs actually use; the platform's own
|
|
537
564
|
* per-run and monthly limits stop a run that goes wrong. `holdout.percent`
|
|
538
565
|
* (5 to 20, default 5) is the share of its samples held back to judge each
|
|
@@ -576,6 +603,8 @@ export class Loop {
|
|
|
576
603
|
* `TrainingRuleRunRefusalCode`. When the attempt in flight is waiting for
|
|
577
604
|
* GPUs to compare (`active_run.state` `waiting_gpus`), this books its two
|
|
578
605
|
* comparison machines again at once instead of starting another attempt.
|
|
606
|
+
* With a rebuild on its way (`chain.rebuild`), it starts the rebuild: from
|
|
607
|
+
* the base on every remaining sample, with no wait for new ones.
|
|
579
608
|
*/
|
|
580
609
|
async runPipeline(id) {
|
|
581
610
|
return this._http.fetchPost(`/api/loop/pipelines/${encodeURIComponent(id)}/run`, {});
|
|
@@ -7,6 +7,22 @@ function trainingIdempotencyKey(explicit) {
|
|
|
7
7
|
}
|
|
8
8
|
return key;
|
|
9
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* `start_from` on the wire. The adapters stay in the order given (oldest
|
|
12
|
+
* first: the order they are merged in), and an empty list is refused here
|
|
13
|
+
* rather than sent, since the API refuses it too.
|
|
14
|
+
*/
|
|
15
|
+
function startFromBody(sf) {
|
|
16
|
+
if (!Array.isArray(sf.adapters) || sf.adapters.length === 0) {
|
|
17
|
+
throw new Error('RunBiOS: startFrom.adapters must name at least one adapter checkpoint, oldest first');
|
|
18
|
+
}
|
|
19
|
+
const body = {
|
|
20
|
+
adapters: sf.adapters.map((a) => ({ job_id: a.jobId, checkpoint_id: a.checkpointId })),
|
|
21
|
+
};
|
|
22
|
+
if (sf.exportMerged !== undefined)
|
|
23
|
+
body.export_merged = sf.exportMerged;
|
|
24
|
+
return body;
|
|
25
|
+
}
|
|
10
26
|
function buildTrainingRequest(params) {
|
|
11
27
|
let trainingMethod = params.method;
|
|
12
28
|
// Legacy alias from SDK versions that modeled VLM training as a method;
|
|
@@ -80,6 +96,8 @@ function buildTrainingRequest(params) {
|
|
|
80
96
|
body.dataset_mixing = params.datasetMixing;
|
|
81
97
|
if (params.mixing !== undefined)
|
|
82
98
|
body.mixing = params.mixing;
|
|
99
|
+
if (params.startFrom !== undefined)
|
|
100
|
+
body.start_from = startFromBody(params.startFrom);
|
|
83
101
|
const config = {};
|
|
84
102
|
if (params.epochs !== undefined)
|
|
85
103
|
config.num_train_epochs = params.epochs;
|
package/dist/types.d.ts
CHANGED
|
@@ -547,15 +547,60 @@ export interface GPUChoice {
|
|
|
547
547
|
/** Native market tier; training and deployment capacity are secure-only. */
|
|
548
548
|
tier?: 'secure';
|
|
549
549
|
}
|
|
550
|
+
/**
|
|
551
|
+
* Train a new LoRA adapter on top of earlier LoRA versions of the same base
|
|
552
|
+
* model (`start_from` on create and preflight). Feature-gated: only when
|
|
553
|
+
* `training.capabilities()` lists the `start_from_adapters` feature as
|
|
554
|
+
* enabled; otherwise the API answers 400 TRAINING_FEATURE_DISABLED with the
|
|
555
|
+
* feature's reason.
|
|
556
|
+
*
|
|
557
|
+
* `adapters` are 1 to 64 adapter checkpoints of earlier jobs, OLDEST FIRST:
|
|
558
|
+
* the job trains on the base with every one of them merged into it in that
|
|
559
|
+
* order. Each must be an exact adapter checkpoint of a completed or stopped
|
|
560
|
+
* LoRA job in the same workspace, on the same pinned model and revision, and
|
|
561
|
+
* each earlier job must itself have started from exactly the adapters before
|
|
562
|
+
* it. The method must be sft or pt and the adapter lora.
|
|
563
|
+
*
|
|
564
|
+
* `exportMerged` also writes the base, the adapters and this job's selected
|
|
565
|
+
* adapter merged into one model: a checkpoint of format `merged`, which is
|
|
566
|
+
* what deploys (the job's own adapters are not deployable alone).
|
|
567
|
+
*
|
|
568
|
+
* Refusals, all before anything is charged: 404 for an unknown, deleted or
|
|
569
|
+
* other-workspace checkpoint; 422 START_FROM_BASE_MISMATCH,
|
|
570
|
+
* START_FROM_CHAIN_BROKEN or START_FROM_LINK_UNREADABLE naming the `index`;
|
|
571
|
+
* 422 START_FROM_NOT_SUPPORTED (a model type or qlora that cannot chain).
|
|
572
|
+
* A job trained this way cannot be resumed (422
|
|
573
|
+
* START_FROM_RESUME_UNSUPPORTED): start it again. If the merged model cannot
|
|
574
|
+
* be saved after training, the job fails with MERGED_EXPORT_FAILED.
|
|
575
|
+
*/
|
|
576
|
+
export interface TrainingStartFromParams {
|
|
577
|
+
adapters: {
|
|
578
|
+
jobId: string;
|
|
579
|
+
checkpointId: string;
|
|
580
|
+
}[];
|
|
581
|
+
exportMerged?: boolean;
|
|
582
|
+
}
|
|
583
|
+
/** `start_from` as the API echoes it on create, preflight and the job, with the digest of the chain's exact bytes. */
|
|
584
|
+
export interface TrainingStartFrom {
|
|
585
|
+
adapters: {
|
|
586
|
+
job_id: string;
|
|
587
|
+
checkpoint_id: string;
|
|
588
|
+
}[];
|
|
589
|
+
export_merged: boolean;
|
|
590
|
+
chain_digest?: string;
|
|
591
|
+
}
|
|
550
592
|
/** Parameters for creating a training job. */
|
|
551
593
|
export interface TrainingCreateParams {
|
|
594
|
+
/** Train on top of earlier LoRA versions of the same base. See {@link TrainingStartFromParams}. */
|
|
595
|
+
startFrom?: TrainingStartFromParams;
|
|
552
596
|
/** Stable retry key. Reuse it after a timeout; a new value creates a new job. Auto-generated when omitted. */
|
|
553
597
|
idempotencyKey?: string;
|
|
554
598
|
/**
|
|
555
599
|
* Model id to fine-tune. Must be hosted on Run BiOS — pick from the platform's
|
|
556
600
|
* own catalog (client.models.search); arbitrary Hugging Face ids are
|
|
557
|
-
* rejected.
|
|
558
|
-
*
|
|
601
|
+
* rejected. Every job starts from a catalog base model; `startFrom` (where
|
|
602
|
+
* the platform enables it) trains a new LoRA adapter on top of earlier LoRA
|
|
603
|
+
* versions of that same base.
|
|
559
604
|
*/
|
|
560
605
|
model: string;
|
|
561
606
|
/** Optional model branch, tag, or commit; the API canonicalizes it to an exact commit. */
|
|
@@ -744,6 +789,10 @@ export interface TrainingJob {
|
|
|
744
789
|
created_at?: string;
|
|
745
790
|
billing_started_at?: string;
|
|
746
791
|
resume_from_job_id?: string | null;
|
|
792
|
+
/** The earlier LoRA versions this job trained on top of; absent for a job from the base. See {@link TrainingStartFromParams}. */
|
|
793
|
+
start_from?: TrainingStartFrom;
|
|
794
|
+
/** The job's merged model (a checkpoint of format `merged`) once it is saved; null until then. Only on a job with `start_from`. */
|
|
795
|
+
merged_checkpoint_id?: string | null;
|
|
747
796
|
resume_count?: number;
|
|
748
797
|
can_resume?: boolean;
|
|
749
798
|
has_cached_dataset?: boolean;
|
|
@@ -979,6 +1028,17 @@ export interface TrainingCheckpoint {
|
|
|
979
1028
|
base_model_id: string;
|
|
980
1029
|
base_model_revision: string;
|
|
981
1030
|
created_at: string;
|
|
1031
|
+
/**
|
|
1032
|
+
* What the stored weights are: `adapter`, `full`, or `merged` -- the base
|
|
1033
|
+
* with the job's `start_from` adapters and its selected adapter merged into
|
|
1034
|
+
* one model, named "merged model (step B)", with `step` the adapter step it
|
|
1035
|
+
* was merged from. A merged model is never final, best or pruned, and is
|
|
1036
|
+
* what deploys: an adapter of a job with `start_from` is refused by
|
|
1037
|
+
* deployments (CHAINED_ADAPTER_NOT_DEPLOYABLE). Absent on an older service.
|
|
1038
|
+
*/
|
|
1039
|
+
format?: 'adapter' | 'full' | 'merged';
|
|
1040
|
+
/** On a merged model: the adapter checkpoint it was merged from. */
|
|
1041
|
+
merged_of_checkpoint_id?: string;
|
|
982
1042
|
}
|
|
983
1043
|
/** Training logs response. */
|
|
984
1044
|
export interface TrainingLogs {
|
|
@@ -1245,6 +1305,24 @@ export interface TrainingConfigFieldCapability {
|
|
|
1245
1305
|
adapters?: string[];
|
|
1246
1306
|
description?: string;
|
|
1247
1307
|
}
|
|
1308
|
+
/**
|
|
1309
|
+
* A whole capability of GET /api/training/capabilities that is on or off as a
|
|
1310
|
+
* unit (`features`), such as `start_from_adapters` (training on top of
|
|
1311
|
+
* earlier LoRA versions, `start_from`). Check `enabled` before using it; when
|
|
1312
|
+
* it is off, `disabled_reason` says why, in words for a person.
|
|
1313
|
+
*/
|
|
1314
|
+
export interface TrainingFeature {
|
|
1315
|
+
id: string;
|
|
1316
|
+
name: string;
|
|
1317
|
+
enabled: boolean;
|
|
1318
|
+
disabled_reason?: string;
|
|
1319
|
+
/** The adapter types it works with, e.g. `["lora"]`. */
|
|
1320
|
+
train_types?: string[];
|
|
1321
|
+
/** The most adapters one `start_from` may name. */
|
|
1322
|
+
max_adapters?: number;
|
|
1323
|
+
/** Model types it cannot be used with. */
|
|
1324
|
+
unsupported_model_types?: string[];
|
|
1325
|
+
}
|
|
1248
1326
|
/** Dynamic contract returned by GET /api/training/capabilities. */
|
|
1249
1327
|
export interface TrainingCapabilities {
|
|
1250
1328
|
contract_version: string;
|
|
@@ -1258,6 +1336,14 @@ export interface TrainingCapabilities {
|
|
|
1258
1336
|
when: Record<string, unknown>;
|
|
1259
1337
|
message: string;
|
|
1260
1338
|
}>;
|
|
1339
|
+
/**
|
|
1340
|
+
* Whole capabilities that are on or off as a unit, such as
|
|
1341
|
+
* `start_from_adapters`. Optional: an older training service does not send
|
|
1342
|
+
* it, and a feature it does not list is not available.
|
|
1343
|
+
*/
|
|
1344
|
+
features?: TrainingFeature[];
|
|
1345
|
+
/** The training engine's native Hugging Face architecture classes, when the service sends them. */
|
|
1346
|
+
architectures?: string[];
|
|
1261
1347
|
}
|
|
1262
1348
|
/**
|
|
1263
1349
|
* Wallet balance (`GET /api/billing/wallet`).
|
|
@@ -2869,11 +2955,61 @@ export type TrainingRulePausedReason = 'consent_invalid' | 'insufficient_funds'
|
|
|
2869
2955
|
/**
|
|
2870
2956
|
* Why a run fired, as the run stores it: `rows` (enough new samples),
|
|
2871
2957
|
* `cadence` (the schedule), `both`, `manual` (train now), `challenger` (the
|
|
2872
|
-
* pipeline's challenger model trained
|
|
2873
|
-
* it),
|
|
2874
|
-
*
|
|
2958
|
+
* pipeline's challenger model, trained from its own base on every sample the
|
|
2959
|
+
* attempt before it could train on), `rebuild` (a member deleted a
|
|
2960
|
+
* conversation the live version learned from, so the model is trained again
|
|
2961
|
+
* from the base on every remaining sample), or `variant` (a recipe variant,
|
|
2962
|
+
* retired: the same base model on the same conversations with one training
|
|
2963
|
+
* setting changed; old runs only).
|
|
2964
|
+
*/
|
|
2965
|
+
export type TrainingTrigger = 'rows' | 'cadence' | 'both' | 'manual' | 'variant' | 'challenger' | 'rebuild';
|
|
2966
|
+
/**
|
|
2967
|
+
* What an attempt was trained on top of, and on which samples: `lineage` on a
|
|
2968
|
+
* run, on a pipeline's versions, attempts and attempt in flight, and on each
|
|
2969
|
+
* side of an attempt comparison.
|
|
2970
|
+
*
|
|
2971
|
+
* `trained_from` is `versions` when the attempt trained a new adapter on top
|
|
2972
|
+
* of the live version: the pipeline's pinned base with the versions in
|
|
2973
|
+
* `built_on` (oldest first) merged into it in that order. It is `base` when
|
|
2974
|
+
* it started from the base model, and `from_base_reason` says why:
|
|
2975
|
+
* `first_version`, `challenger` (the second model never builds on another
|
|
2976
|
+
* model's versions), `rebuild` (a deleted conversation, with `rebuild` true;
|
|
2977
|
+
* or, with `rebuild` false, a live version already built on 64 versions, the
|
|
2978
|
+
* most one attempt can build on), `model_cannot_build_on` or
|
|
2979
|
+
* `chain_link_unreadable` (an earlier version could not be read). Null when
|
|
2980
|
+
* it built on versions.
|
|
2981
|
+
*
|
|
2982
|
+
* What it trained on (`train_rows`) is `new_samples` (new since the attempt
|
|
2983
|
+
* before it) + `carried_samples` (trained by the attempts in
|
|
2984
|
+
* `carried_from_attempts`, which did not go live) + `replayed_samples` (about
|
|
2985
|
+
* a fifth of what it trained on, drawn at random from what the versions it
|
|
2986
|
+
* builds on already learned, so it does not forget them). `learned_before` is
|
|
2987
|
+
* how many samples those versions had learned: they are not trained again,
|
|
2988
|
+
* except the replayed ones. An attempt from the base trains on every sample
|
|
2989
|
+
* it can: `new_samples` is all of them, and the other three counts are 0.
|
|
2990
|
+
* The four counts are null on an attempt from before the chain and on every
|
|
2991
|
+
* attempt of a pipeline that does not build on its live version: only
|
|
2992
|
+
* `train_rows` is known then.
|
|
2993
|
+
*
|
|
2994
|
+
* `served_as` is `adapter` for an attempt from the base and `merged` for one
|
|
2995
|
+
* built on versions (one model with every version merged in). A winner is
|
|
2996
|
+
* merged in and served as one model. `learned_deleted_conversation` is true
|
|
2997
|
+
* when a conversation it trained on has since been deleted: a rebuild
|
|
2998
|
+
* without it is on its way, and it can no longer be rolled back to.
|
|
2875
2999
|
*/
|
|
2876
|
-
export
|
|
3000
|
+
export interface Lineage {
|
|
3001
|
+
trained_from: 'base' | 'versions';
|
|
3002
|
+
built_on: number[];
|
|
3003
|
+
served_as: 'adapter' | 'merged';
|
|
3004
|
+
new_samples: number | null;
|
|
3005
|
+
carried_samples: number | null;
|
|
3006
|
+
carried_from_attempts: number[];
|
|
3007
|
+
replayed_samples: number | null;
|
|
3008
|
+
learned_before: number | null;
|
|
3009
|
+
rebuild: boolean;
|
|
3010
|
+
learned_deleted_conversation: boolean;
|
|
3011
|
+
from_base_reason: 'first_version' | 'challenger' | 'rebuild' | 'model_cannot_build_on' | 'chain_link_unreadable' | null;
|
|
3012
|
+
}
|
|
2877
3013
|
/**
|
|
2878
3014
|
* The four training settings a recipe variant may change, as the run was
|
|
2879
3015
|
* actually trained: the training service's own defaults are filled in where
|
|
@@ -2968,6 +3104,39 @@ export interface TrainingRunRejectRequest {
|
|
|
2968
3104
|
export interface TrainingRunRollbackRequest {
|
|
2969
3105
|
reason?: string;
|
|
2970
3106
|
}
|
|
3107
|
+
/**
|
|
3108
|
+
* The machine codes `rollbackTrainingRun` can be refused with.
|
|
3109
|
+
*
|
|
3110
|
+
* - `NOT_PROMOTED` (409): the run never went live.
|
|
3111
|
+
* - `ROLLBACK_WINDOW_CLOSED` (409): more than 30 days since it went live.
|
|
3112
|
+
* - `ROLLBACK_SUPERSEDED` (409): a later version went live since; roll that
|
|
3113
|
+
* one back.
|
|
3114
|
+
* - `NOTHING_TO_RESTORE` (409): nothing is recorded as serving before it.
|
|
3115
|
+
* - `RUN_ACTIVE` (409): an attempt is being compared with this version, or is
|
|
3116
|
+
* being built on top of it. Cancel that attempt (`cancelTrainingRun`)
|
|
3117
|
+
* first, or wait for it to finish.
|
|
3118
|
+
* - `ROLLBACK_DELETED_DATA` (409): the version it would put back learned from
|
|
3119
|
+
* a conversation a member deleted since, so it never answers again; a
|
|
3120
|
+
* rebuild without that conversation replaces it.
|
|
3121
|
+
*/
|
|
3122
|
+
export type TrainingRunRollbackRefusalCode = 'NOT_PROMOTED' | 'ROLLBACK_WINDOW_CLOSED' | 'ROLLBACK_SUPERSEDED' | 'NOTHING_TO_RESTORE' | 'RUN_ACTIVE' | 'ROLLBACK_DELETED_DATA';
|
|
3123
|
+
/** A pipeline whose model a trace delete rebuilds: its live version learned from the deleted conversation. */
|
|
3124
|
+
export interface LoopTraceRebuild {
|
|
3125
|
+
pipeline_id: string;
|
|
3126
|
+
pipeline_name: string;
|
|
3127
|
+
}
|
|
3128
|
+
/**
|
|
3129
|
+
* `DELETE /api/loop/traces/{id}`. `rebuilds` names every pipeline whose live
|
|
3130
|
+
* version learned from the conversation: each rebuilds its model from the
|
|
3131
|
+
* base without it. A pipeline that is switched off is named too; its rebuild
|
|
3132
|
+
* starts when it is switched back on. Empty when none did; absent on an
|
|
3133
|
+
* older service.
|
|
3134
|
+
*/
|
|
3135
|
+
export interface LoopTraceDeleteResponse {
|
|
3136
|
+
deleted: boolean;
|
|
3137
|
+
trace_id: string;
|
|
3138
|
+
rebuilds?: LoopTraceRebuild[];
|
|
3139
|
+
}
|
|
2971
3140
|
export interface TrainingRunCancelRequest {
|
|
2972
3141
|
reason?: string;
|
|
2973
3142
|
}
|
|
@@ -3004,6 +3173,13 @@ export interface TrainingRun {
|
|
|
3004
3173
|
attempts: number;
|
|
3005
3174
|
next_poll_at: string | null;
|
|
3006
3175
|
last_reason: string | null;
|
|
3176
|
+
/**
|
|
3177
|
+
* Why it failed or was set aside, as a code. Two belong to a pipeline that
|
|
3178
|
+
* builds on its live version: `CHAIN_MOVED` (it was built on a version that
|
|
3179
|
+
* was rolled back while it trained; it is superseded, no comparison was
|
|
3180
|
+
* paid for, and its samples go to the next attempt) and `NO_MERGED_EXPORT`
|
|
3181
|
+
* (its training finished without the merged model it is served as).
|
|
3182
|
+
*/
|
|
3007
3183
|
error_code: string | null;
|
|
3008
3184
|
last_error: string | null;
|
|
3009
3185
|
/**
|
|
@@ -3030,6 +3206,8 @@ export interface TrainingRun {
|
|
|
3030
3206
|
spent_eval_cents: number;
|
|
3031
3207
|
train_rows: number | null;
|
|
3032
3208
|
holdout_rows: number | null;
|
|
3209
|
+
/** What it was trained on top of, and on which samples. See {@link Lineage}. */
|
|
3210
|
+
lineage: Lineage;
|
|
3033
3211
|
train_dataset_id: string | null;
|
|
3034
3212
|
/** Null on a run made since the held-back rows stopped leaving the loop; set on older runs. */
|
|
3035
3213
|
holdout_dataset_id: string | null;
|
|
@@ -3587,6 +3765,13 @@ export interface BenchmarkRun {
|
|
|
3587
3765
|
prompt_tokens: number;
|
|
3588
3766
|
completion_tokens: number;
|
|
3589
3767
|
spent_cents: number;
|
|
3768
|
+
/**
|
|
3769
|
+
* How many of the conversations this replay asked the version under test
|
|
3770
|
+
* learned -- trained on itself, or learned by a version it was built on
|
|
3771
|
+
* ("N of these conversations were learned by this version"): a score on
|
|
3772
|
+
* those is partly recall. Null for a version the merge chain did not build.
|
|
3773
|
+
*/
|
|
3774
|
+
learned_by_version: number | null;
|
|
3590
3775
|
status: BenchmarkRunStatus;
|
|
3591
3776
|
/** The whole explanation when there is no score, so never empty on one. */
|
|
3592
3777
|
status_reason: string | null;
|
|
@@ -3619,6 +3804,8 @@ export interface BenchmarkHistoryPoint {
|
|
|
3619
3804
|
incumbent_pass_rate: number | null;
|
|
3620
3805
|
candidate_pass_rate: number | null;
|
|
3621
3806
|
checkpoint_id: string | null;
|
|
3807
|
+
/** The replay's {@link BenchmarkRun.learned_by_version}. */
|
|
3808
|
+
learned_by_version: number | null;
|
|
3622
3809
|
decision: TrainingDecision | null;
|
|
3623
3810
|
status: BenchmarkRunStatus;
|
|
3624
3811
|
status_reason: string | null;
|
|
@@ -3753,7 +3940,10 @@ export interface BenchmarkRetireParams {
|
|
|
3753
3940
|
* it (`paused_reason` says why).
|
|
3754
3941
|
* - `RUN_ACTIVE` (409): one attempt at a time, so two cannot race to promote.
|
|
3755
3942
|
* - `VERSION_LIMIT_REACHED` (409): the pipeline has made `max_versions`
|
|
3756
|
-
* versions. Raise the limit or remove it.
|
|
3943
|
+
* versions and owes no rebuild. Raise the limit or remove it. A rebuild
|
|
3944
|
+
* after a deleted conversation (`chain.rebuild`) is not held by the limit:
|
|
3945
|
+
* Train now starts it, and each one that goes live is a version, so the
|
|
3946
|
+
* pipeline can end past its limit.
|
|
3757
3947
|
* - `MONTHLY_LIMIT_REACHED` (409): the most this month's attempts can have
|
|
3758
3948
|
* cost plus the most one more may cost would pass the pipeline's internal
|
|
3759
3949
|
* monthly limit. The body carries the figures:
|
|
@@ -3919,6 +4109,8 @@ export interface PipelineActiveRun {
|
|
|
3919
4109
|
last_error: string | null;
|
|
3920
4110
|
train_rows: number | null;
|
|
3921
4111
|
holdout_rows: number | null;
|
|
4112
|
+
/** What it trains on top of, and on which samples. See {@link Lineage}. */
|
|
4113
|
+
lineage: Lineage;
|
|
3922
4114
|
/**
|
|
3923
4115
|
* Its comparison scored at least one conversation. False on a reported run
|
|
3924
4116
|
* means the held-back set was too small to compare anything: that run is
|
|
@@ -3953,6 +4145,8 @@ export interface PipelineVersion {
|
|
|
3953
4145
|
checkpoint_id: string | null;
|
|
3954
4146
|
train_rows: number | null;
|
|
3955
4147
|
holdout_rows: number | null;
|
|
4148
|
+
/** What it was trained on top of, and on which samples. See {@link Lineage}. */
|
|
4149
|
+
lineage: Lineage;
|
|
3956
4150
|
/** Head-to-head against the champion of its day, on conversations neither trained on. */
|
|
3957
4151
|
rows_scored: number | null;
|
|
3958
4152
|
win_rate: number | null;
|
|
@@ -4020,10 +4214,12 @@ export interface PipelineVersion {
|
|
|
4020
4214
|
*/
|
|
4021
4215
|
/**
|
|
4022
4216
|
* Why an attempt was made: new data (`data`, a recipe variant included), the
|
|
4023
|
-
* schedule (`schedule`), a member's train-now (`manual`),
|
|
4024
|
-
* model trained
|
|
4217
|
+
* schedule (`schedule`), a member's train-now (`manual`), the challenger
|
|
4218
|
+
* model trained from its own base on all samples (`challenger`), or a rebuild
|
|
4219
|
+
* from the base after a member deleted a conversation the live version
|
|
4220
|
+
* learned from (`rebuild`).
|
|
4025
4221
|
*/
|
|
4026
|
-
export type PipelineAttemptTrigger = 'data' | 'schedule' | 'manual' | 'challenger';
|
|
4222
|
+
export type PipelineAttemptTrigger = 'data' | 'schedule' | 'manual' | 'challenger' | 'rebuild';
|
|
4027
4223
|
/** What became of an attempt. */
|
|
4028
4224
|
export type PipelineAttemptOutcome = 'became_version' | 'not_better' | 'inconclusive' | 'stopped' | 'failed' | 'running' | 'waiting_for_review';
|
|
4029
4225
|
/** One attempt: every training run a pipeline makes, winner or not. */
|
|
@@ -4043,6 +4239,8 @@ export interface PipelineAttempt {
|
|
|
4043
4239
|
/** The samples its set trained on and held back; null until its set was built. */
|
|
4044
4240
|
train_rows: number | null;
|
|
4045
4241
|
holdout_rows: number | null;
|
|
4242
|
+
/** What it was trained on top of, and on which samples. See {@link Lineage}. */
|
|
4243
|
+
lineage: Lineage;
|
|
4046
4244
|
/** The version it became, when it became one. */
|
|
4047
4245
|
version: number | null;
|
|
4048
4246
|
created_at: string;
|
|
@@ -4072,7 +4270,10 @@ export interface PipelineModelRef {
|
|
|
4072
4270
|
/**
|
|
4073
4271
|
* When a new version takes over. `holdout` (the default) when it wins on the
|
|
4074
4272
|
* held-back set; `all`, `primary`, `k_of_n` (with `k`) and `weighted` also
|
|
4075
|
-
* weigh the pipeline's benchmarks; `manual` never on its own.
|
|
4273
|
+
* weigh the pipeline's benchmarks; `manual` never on its own. A rebuild after
|
|
4274
|
+
* a deleted conversation (trigger `rebuild`) is the exception under every
|
|
4275
|
+
* policy but `manual`: it goes live unless it is clearly worse than the live
|
|
4276
|
+
* version. Under `manual` it waits for a member like any attempt.
|
|
4076
4277
|
*/
|
|
4077
4278
|
export interface PipelinePromotion {
|
|
4078
4279
|
policy: 'holdout' | 'all' | 'primary' | 'k_of_n' | 'weighted' | 'manual';
|
|
@@ -4123,8 +4324,8 @@ export interface PipelineHoldout {
|
|
|
4123
4324
|
* `first_version`: `needed` usable samples the first attempt needs -- the
|
|
4124
4325
|
* rows it trains on plus the ones held back to judge it. `new_rows`: `needed`
|
|
4125
4326
|
* usable samples the last trained attempt did not have, which is
|
|
4126
|
-
* `floors.new_samples`
|
|
4127
|
-
*
|
|
4327
|
+
* `floors.new_samples` (1,000), however many rows that attempt trained on.
|
|
4328
|
+
* `have`, `short` and `message` are
|
|
4128
4329
|
* measured on the detail only. There, after an attempt has trained, `short`
|
|
4129
4330
|
* is how many more new samples have to arrive: new samples fill the held-back
|
|
4130
4331
|
* set first, so after the held-back share is raised it can be more than
|
|
@@ -4156,6 +4357,34 @@ export interface PipelineLiveDeployment {
|
|
|
4156
4357
|
status: PipelineLiveDeploymentStatus;
|
|
4157
4358
|
checked_at: string;
|
|
4158
4359
|
}
|
|
4360
|
+
/**
|
|
4361
|
+
* A rebuild on its way: a member deleted `conversations` conversations the
|
|
4362
|
+
* live version learned from, first at `since`. The next attempt trains from
|
|
4363
|
+
* the base on every remaining sample without waiting for new ones; until it
|
|
4364
|
+
* goes live, the live version still holds what it learned from them.
|
|
4365
|
+
*/
|
|
4366
|
+
export interface PipelineChainRebuild {
|
|
4367
|
+
reason: 'conversation_deleted';
|
|
4368
|
+
conversations: number;
|
|
4369
|
+
since: string;
|
|
4370
|
+
}
|
|
4371
|
+
/**
|
|
4372
|
+
* The live version as what it is made of: the pinned base with `versions`
|
|
4373
|
+
* merged into it in that order, the live one last. Null when nothing this
|
|
4374
|
+
* pipeline trained is live on its current model, and when the pipeline does
|
|
4375
|
+
* not build on its live version (then every attempt trains from the base on
|
|
4376
|
+
* all samples). `learned_samples` is how many samples those versions learned
|
|
4377
|
+
* (the next attempt does not train them again, except a replayed fifth);
|
|
4378
|
+
* `not_learned_samples` is how many were trained by attempts that did not go
|
|
4379
|
+
* live, so the next attempt trains them again. Null on the list.
|
|
4380
|
+
*/
|
|
4381
|
+
export interface PipelineChain {
|
|
4382
|
+
versions: number[];
|
|
4383
|
+
live_version: number;
|
|
4384
|
+
learned_samples: number;
|
|
4385
|
+
not_learned_samples: number | null;
|
|
4386
|
+
rebuild: PipelineChainRebuild | null;
|
|
4387
|
+
}
|
|
4159
4388
|
export interface Pipeline {
|
|
4160
4389
|
rule_id: string;
|
|
4161
4390
|
rule_name: string;
|
|
@@ -4173,8 +4402,9 @@ export interface Pipeline {
|
|
|
4173
4402
|
/** A conversation carrying ANY of these tags is this pipeline's data. */
|
|
4174
4403
|
tags: string[];
|
|
4175
4404
|
/**
|
|
4176
|
-
* A second base model trained on
|
|
4177
|
-
*
|
|
4405
|
+
* A second base model, trained from its own base on all samples after each
|
|
4406
|
+
* attempt (it never builds on this pipeline's versions, which are another
|
|
4407
|
+
* model); it becomes the next version only if it beats the live one, and then it
|
|
4178
4408
|
* becomes the pipeline's model (`base_model_id`) while the model it replaced
|
|
4179
4409
|
* becomes this one. `revision` is the commit the server pinned.
|
|
4180
4410
|
*/
|
|
@@ -4185,7 +4415,11 @@ export interface Pipeline {
|
|
|
4185
4415
|
/** Rows the next version holds back and never trains on. Detail only; null on the list. */
|
|
4186
4416
|
holdout_size: number | null;
|
|
4187
4417
|
minimums: PipelineMinimums;
|
|
4188
|
-
/**
|
|
4418
|
+
/**
|
|
4419
|
+
* The pinned base every version is trained over: from it, or, where the
|
|
4420
|
+
* pipeline builds on its live version (`chain`), on top of the versions
|
|
4421
|
+
* before it merged into it.
|
|
4422
|
+
*/
|
|
4189
4423
|
base_model_id: string;
|
|
4190
4424
|
base_model_revision: string;
|
|
4191
4425
|
/** The version serving now; null when what serves is not a version of this pipeline. */
|
|
@@ -4198,6 +4432,8 @@ export interface Pipeline {
|
|
|
4198
4432
|
* serves is not a deployment, or before the first check of that deployment.
|
|
4199
4433
|
*/
|
|
4200
4434
|
live_deployment: PipelineLiveDeployment | null;
|
|
4435
|
+
/** The live version as the versions merged into it. See {@link PipelineChain}. */
|
|
4436
|
+
chain: PipelineChain | null;
|
|
4201
4437
|
/** What the pipeline's history takes up in storage. */
|
|
4202
4438
|
storage: LoopStorage;
|
|
4203
4439
|
/** The name the app calls. It does not change when a version is promoted. */
|
|
@@ -4513,13 +4749,11 @@ export interface PipelineFloors {
|
|
|
4513
4749
|
* back more needs more; its own `minimums` says how many).
|
|
4514
4750
|
*/
|
|
4515
4751
|
first_version_samples: number;
|
|
4516
|
-
/** The fewest new samples any later attempt waits for. */
|
|
4517
|
-
new_samples: number;
|
|
4518
4752
|
/**
|
|
4519
|
-
* The
|
|
4520
|
-
*
|
|
4753
|
+
* The new samples every later attempt waits for, counted since the last
|
|
4754
|
+
* trained attempt, however many rows that attempt trained on.
|
|
4521
4755
|
*/
|
|
4522
|
-
|
|
4756
|
+
new_samples: number;
|
|
4523
4757
|
/** The held-back rule a pipeline chooses its share within. */
|
|
4524
4758
|
held_back: HeldBackFloors;
|
|
4525
4759
|
}
|
|
@@ -4613,6 +4847,8 @@ export interface VersionBenchmarkScore {
|
|
|
4613
4847
|
rows_scored: number | null;
|
|
4614
4848
|
rows_total: number | null;
|
|
4615
4849
|
items_digest: string | null;
|
|
4850
|
+
/** The replay's {@link BenchmarkRun.learned_by_version}; null when not measured. */
|
|
4851
|
+
learned_by_version: number | null;
|
|
4616
4852
|
comparable: boolean;
|
|
4617
4853
|
why: string | null;
|
|
4618
4854
|
/** The share each side passed (score 0.70 or more); null when there is no score. */
|
|
@@ -4735,6 +4971,8 @@ export interface AttemptCompareSide {
|
|
|
4735
4971
|
decision: TrainingDecision | null;
|
|
4736
4972
|
train_rows: number | null;
|
|
4737
4973
|
holdout_rows: number | null;
|
|
4974
|
+
/** What it was trained on top of, and on which samples. See {@link Lineage}. */
|
|
4975
|
+
lineage: Lineage;
|
|
4738
4976
|
finished_at: string | null;
|
|
4739
4977
|
holdout: AttemptHoldout;
|
|
4740
4978
|
}
|