runbios-sdk 0.2.22 → 0.3.0-dev.322
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/README.md +48 -1
- package/dist/client.d.ts +11 -0
- package/dist/client.js +29 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +2 -2
- package/dist/resources/inference.d.ts +68 -2
- package/dist/resources/inference.js +71 -5
- package/dist/resources/loop.d.ts +63 -71
- package/dist/resources/loop.js +112 -18
- package/dist/resources/training.js +12 -2
- package/dist/types.d.ts +209 -69
- package/package.json +3 -2
package/dist/resources/loop.d.ts
CHANGED
|
@@ -1,63 +1,11 @@
|
|
|
1
|
-
import type
|
|
2
|
-
import type { LoopTrace, LoopTraceDeleteResponse, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopImportParams, LoopImportResult, LoopSignal, LoopSignalParams, LoopLabel, LoopLabelCount, LoopConfig, LoopConfigParams, LoopStats, LoopTracePipelinesResponse, LoopMetricsHistory, LoopMetricsHistoryParams, TrainingRuleDeleteResponse, TrainingRunListParams, TrainingRunListResponse, TrainingRunResponse, TrainingRunActionResponse, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, Evaluation, EvaluationItemListParams, EvaluationItemsResponse, Benchmark, BenchmarkCreateParams, BenchmarkHistoryParams, BenchmarkHistoryResponse, BenchmarkItemListParams, BenchmarkItemsResponse, BenchmarkListParams, BenchmarkListResponse, BenchmarkRetireParams, BenchmarkRun, BenchmarkRunItemListParams, BenchmarkRunItemsResponse, Pipeline, PipelineListParams, PipelineListResponse, PipelineWriteRequest, PipelineMutationResponse, LoopModelsResponse, PipelineImportParams, PipelineBenchmarksResponse, PipelineBenchmarkAttachParams, PipelineBenchmarkMoveParams, VersionCompareResponse, VersionCompareParams, AttemptCompareResponse, AttemptCompareParams } from '../types.js';
|
|
3
|
-
/**
|
|
4
|
-
* The Conscious Loop -- pipelines that keep improving a model from the
|
|
5
|
-
* conversations you send them.
|
|
6
|
-
*
|
|
7
|
-
* A pipeline is one use case: a model, the tags whose samples train it, when
|
|
8
|
-
* to train, and how a new version is promoted. Every time it trains it makes
|
|
9
|
-
* an attempt; an attempt that goes live becomes the next version: one that
|
|
10
|
-
* beats the live model, one a member approves, or, after a deleted
|
|
11
|
-
* conversation, a rebuild without it that is not clearly worse (a `manual`
|
|
12
|
-
* pipeline waits for a member even then).
|
|
13
|
-
*
|
|
14
|
-
* Nothing is captured until a source is switched on, and you can switch it
|
|
15
|
-
* off again at any time. Creating a pipeline switches on the source named by
|
|
16
|
-
* its own tag (`own_tag`), stamping that tag, and deleting it switches that
|
|
17
|
-
* source off again. Anything that looks like a credential or a personal detail
|
|
18
|
-
* is removed before the record is written, never afterwards.
|
|
19
|
-
*
|
|
20
|
-
* Requires an API key carrying `loop:read` and/or `loop:write`. Those scopes are
|
|
21
|
-
* deliberately absent from the Read Only preset, because what they return is
|
|
22
|
-
* your raw prompts and completions rather than catalog metadata.
|
|
23
|
-
*
|
|
24
|
-
* Offered on the development environment only for now. Anywhere else every
|
|
25
|
-
* method here, reads included, rejects with a `ComingSoonError` whose
|
|
26
|
-
* `code` is `LOOP_COMING_SOON` (HTTP 403): a deliberate product state, not an
|
|
27
|
-
* outage, so retrying cannot succeed there.
|
|
28
|
-
*
|
|
29
|
-
* @example
|
|
30
|
-
* ```ts
|
|
31
|
-
* // 1. A pipeline: a model it can train, and its own tag.
|
|
32
|
-
* const { models } = await client.loop.listModels();
|
|
33
|
-
* const { pipeline } = await client.loop.createPipeline({
|
|
34
|
-
* name: 'Support replies',
|
|
35
|
-
* model: { id: models[0].id },
|
|
36
|
-
* });
|
|
37
|
-
*
|
|
38
|
-
* // 2. Send samples from your app to the pipeline's own source: its create
|
|
39
|
-
* // switched that source on, and it tags everything it records.
|
|
40
|
-
* const sent = await client.loop.capture({
|
|
41
|
-
* deployment_id: pipeline.own_tag!,
|
|
42
|
-
* model: 'gpt-4o',
|
|
43
|
-
* messages: [{ role: 'user', content: 'what is our refund window' }],
|
|
44
|
-
* completion: 'thirty days',
|
|
45
|
-
* });
|
|
46
|
-
*
|
|
47
|
-
* // 3. Say whether it was right, on the trace_id the capture answered with.
|
|
48
|
-
* // A correction is the most valuable answer.
|
|
49
|
-
* if (sent.trace_id) {
|
|
50
|
-
* await client.loop.correct(sent.trace_id, 'Thirty days from delivery, no questions asked.');
|
|
51
|
-
* }
|
|
52
|
-
*
|
|
53
|
-
* // 4. Once it has its minimum, it trains; versions appear on the pipeline.
|
|
54
|
-
* const p = await client.loop.getPipeline(pipeline.rule_id);
|
|
55
|
-
* ```
|
|
56
|
-
*/
|
|
1
|
+
import { type HttpClient } from '../client.js';
|
|
2
|
+
import type { LoopTrace, LoopTraceDeleteResponse, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopImportParams, LoopImportResult, LoopSignal, LoopSignalParams, LoopLabel, LoopLabelCount, LoopConfig, LoopConfigParams, LoopStats, LoopTracePipelinesResponse, LoopMetricsHistory, LoopMetricsHistoryParams, TrainingRuleDeleteResponse, TrainingRunListParams, TrainingRunListResponse, TrainingRunResponse, TrainingRunActionResponse, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, Evaluation, EvaluationItemListParams, EvaluationItemsResponse, Benchmark, BenchmarkCreateParams, BenchmarkHistoryParams, BenchmarkHistoryResponse, BenchmarkItemListParams, BenchmarkItemsResponse, BenchmarkListParams, BenchmarkListResponse, BenchmarkRetireParams, BenchmarkRun, BenchmarkRunItemListParams, BenchmarkRunItemsResponse, Pipeline, PipelineAttemptListParams, PipelineAttemptListResponse, PipelineListParams, PipelineListResponse, PipelineWriteRequest, PipelineMutationResponse, LoopModelsResponse, PipelineImportParams, PipelineBenchmarksResponse, PipelineBenchmarkAttachParams, PipelineBenchmarkMoveParams, VersionCompareResponse, VersionCompareParams, AttemptCompareResponse, AttemptCompareParams } from '../types.js';
|
|
57
3
|
export declare class Loop {
|
|
58
4
|
private readonly _http;
|
|
59
5
|
/** @internal */
|
|
60
6
|
constructor(_http: HttpClient);
|
|
7
|
+
/** @internal How feedback waits between tries; tests replace it so nothing sleeps. */
|
|
8
|
+
_sleep: (ms: number) => Promise<void>;
|
|
61
9
|
/**
|
|
62
10
|
* Send one exchange to be captured.
|
|
63
11
|
*
|
|
@@ -139,7 +87,14 @@ export declare class Loop {
|
|
|
139
87
|
* a shorter list moves every row after the gap and imports it again.
|
|
140
88
|
*/
|
|
141
89
|
importRows(params: LoopImportParams): Promise<LoopImportResult>;
|
|
142
|
-
/**
|
|
90
|
+
/**
|
|
91
|
+
* List captured conversations, newest first.
|
|
92
|
+
*
|
|
93
|
+
* `q` searches the text: 2 to 200 characters, any case, matched as written
|
|
94
|
+
* in every message, the answer and its tool calls. Samples kept in your own
|
|
95
|
+
* storage cannot be searched; a search answers how many were left out as
|
|
96
|
+
* `unsearched`.
|
|
97
|
+
*/
|
|
143
98
|
listTraces(params?: LoopTraceListParams): Promise<LoopTraceListResponse>;
|
|
144
99
|
/**
|
|
145
100
|
* Read one conversation, with every verdict and tag recorded on it.
|
|
@@ -176,6 +131,13 @@ export declare class Loop {
|
|
|
176
131
|
* `edited` or `gold` verdict is the example, in place of the model's answer;
|
|
177
132
|
* a `rejected` answer is left out. Nothing is learned from an answer that was
|
|
178
133
|
* rejected or replaced. `source` defaults to `human`.
|
|
134
|
+
*
|
|
135
|
+
* Feedback on an `X-Loop-Sample-Id` sent right after the answer can reach
|
|
136
|
+
* the service before the sample does, which answers 409
|
|
137
|
+
* `SAMPLE_NOT_YET_RECORDED` and stores nothing. This method, and so `upvote`,
|
|
138
|
+
* `downvote`, `correct` and `gold`, sends it again after the `Retry-After`
|
|
139
|
+
* seconds (2 when none is given), at most 6 times in all, and then throws
|
|
140
|
+
* that error. No other error is retried.
|
|
179
141
|
*/
|
|
180
142
|
signal(traceId: string, params: LoopSignalParams): Promise<LoopSignal>;
|
|
181
143
|
/**
|
|
@@ -356,15 +318,21 @@ export declare class Loop {
|
|
|
356
318
|
*/
|
|
357
319
|
cancelTrainingRun(id: string, params?: TrainingRunCancelRequest): Promise<TrainingRunActionResponse>;
|
|
358
320
|
/**
|
|
359
|
-
*
|
|
360
|
-
* the live model),
|
|
361
|
-
*
|
|
321
|
+
* The workspace's pipelines, a page at a time: its versions (the attempts
|
|
322
|
+
* that became the live model), its latest 5 attempts (`attempts_total` says
|
|
323
|
+
* how many; {@link listPipelineAttempts} pages through every one), which
|
|
324
|
+
* version is live, the attempt in flight, and what the pipeline is waiting
|
|
325
|
+
* for.
|
|
326
|
+
*
|
|
327
|
+
* `limit` is 100 when absent and at most 100; `q` matches the name, the tag
|
|
328
|
+
* or the model (case-insensitive); `status` keeps the pipelines `training`,
|
|
329
|
+
* waiting on you (`needs_you`), `serving` or `paused`.
|
|
362
330
|
*
|
|
363
331
|
* RETURNED WITH ITS ENVELOPE, because the list is not always all of them:
|
|
364
|
-
* the service returns
|
|
365
|
-
*
|
|
366
|
-
*
|
|
367
|
-
*
|
|
332
|
+
* the service returns one page, newest first, `total` is every pipeline
|
|
333
|
+
* that matches and `truncated` is true when more exist after this page. A
|
|
334
|
+
* caller that shows `pipelines` alone presents a short list as the whole of
|
|
335
|
+
* it; the next page is `{ offset: offset + pipelines.length }`.
|
|
368
336
|
*/
|
|
369
337
|
listPipelines(params?: PipelineListParams): Promise<PipelineListResponse>;
|
|
370
338
|
/**
|
|
@@ -373,11 +341,21 @@ export declare class Loop {
|
|
|
373
341
|
* `status` says whether it is the platform working (`running`), the member
|
|
374
342
|
* who has to act (`needs_review`, `needs_funds`), or nothing at all until
|
|
375
343
|
* somebody does (`paused`, with `paused_reason` saying why; `complete` at
|
|
376
|
-
* `max_versions`).
|
|
377
|
-
*
|
|
378
|
-
* at most, not an exact spend.
|
|
344
|
+
* `max_versions`). The monthly safety limit is the platform's and has no
|
|
345
|
+
* field: when it holds the pipeline, `next_reason` says so.
|
|
379
346
|
*/
|
|
380
347
|
getPipeline(id: string): Promise<Pipeline>;
|
|
348
|
+
/**
|
|
349
|
+
* A pipeline's attempts, newest first, a page at a time: every training run
|
|
350
|
+
* it made, winner or not, each the shape `attempts` has on the pipeline
|
|
351
|
+
* (with its `explanation` in plain words). `limit` is 50 when absent and at
|
|
352
|
+
* most 200. Narrow by `result` (`became_version`, `not_better`,
|
|
353
|
+
* `inconclusive`, `failed`, `stopped`, `running` -- which includes one
|
|
354
|
+
* waiting for a decision -- or `waiting_for_review`) and by `model` (`main`,
|
|
355
|
+
* or `second` for the second model's attempts); `total` follows the
|
|
356
|
+
* filters.
|
|
357
|
+
*/
|
|
358
|
+
listPipelineAttempts(id: string, params?: PipelineAttemptListParams): Promise<PipelineAttemptListResponse>;
|
|
381
359
|
/**
|
|
382
360
|
* The base models a pipeline can train: only those the engine trains with
|
|
383
361
|
* LoRA SFT and the platform can serve for the comparison, recommended first.
|
|
@@ -391,9 +369,12 @@ export declare class Loop {
|
|
|
391
369
|
* judged on at most `scored_rows` of them). They are read-only.
|
|
392
370
|
* Where a pipeline builds on its live version (its `chain` is set), each
|
|
393
371
|
* attempt trains a new adapter on top of the live version, on what it has
|
|
394
|
-
* not learned yet plus a replay of what it already learned (about
|
|
372
|
+
* not learned yet plus a replay of what it already learned (about half
|
|
395
373
|
* of what it trains on, picked at random, so it is not forgotten), and a
|
|
396
374
|
* winner is merged in and served as one model.
|
|
375
|
+
* `unavailable` lists every other model the platform knows, each with
|
|
376
|
+
* why it is not offered here (`reason_code`, and `reason`, one plain
|
|
377
|
+
* sentence); `[]` from a service from before it was sent.
|
|
397
378
|
* When the list cannot be read it throws `ApiError` 503
|
|
398
379
|
* `MODELS_UNAVAILABLE`, whose `body.floors` still carries the floors.
|
|
399
380
|
*/
|
|
@@ -408,12 +389,14 @@ export declare class Loop {
|
|
|
408
389
|
* Where the pipeline builds on its live version (`chain`), each attempt
|
|
409
390
|
* waits for 1,000 new samples and trains a new adapter on top of the live
|
|
410
391
|
* version, on what it has not learned plus a replay of what it already
|
|
411
|
-
* learned (about
|
|
392
|
+
* learned (about half of what it trains on, picked at random); a winner
|
|
412
393
|
* is merged in.
|
|
413
394
|
* What is charged is the GPU time the runs actually use; the platform's own
|
|
414
395
|
* per-run and monthly limits stop a run that goes wrong. `holdout.percent`
|
|
415
396
|
* (5 to 20, default 5) is the share of its samples held back to judge each
|
|
416
|
-
* attempt and never trained on.
|
|
397
|
+
* attempt and never trained on. `training` sets any of its training
|
|
398
|
+
* settings (`epochs`, `learning_rate`, `lora_rank`, `lora_alpha`,
|
|
399
|
+
* `replay_percent`); left out, or null, every one is automatic.
|
|
417
400
|
*/
|
|
418
401
|
createPipeline(params: PipelineWriteRequest): Promise<PipelineMutationResponse>;
|
|
419
402
|
/**
|
|
@@ -421,7 +404,13 @@ export declare class Loop {
|
|
|
421
404
|
* the limit and `challenger_model: null` removes the challenger. `promotion`
|
|
422
405
|
* is how the promotion policy is set. A new `holdout.percent` is a new
|
|
423
406
|
* revision: lowering it keeps every sample already held back, and raising it
|
|
424
|
-
* holds back only new samples.
|
|
407
|
+
* holds back only new samples. `training` changes only the settings it
|
|
408
|
+
* carries -- `{ epochs: 8 }` sets the passes and keeps the rest,
|
|
409
|
+
* `{ epochs: null }` puts the passes back to automatic -- and
|
|
410
|
+
* `training: null` puts every one back to automatic; the next attempt
|
|
411
|
+
* trains with them, and earlier attempts keep the settings they trained
|
|
412
|
+
* with. Out of range is 422 `TRAINING_SETTING_OUT_OF_RANGE`, naming the
|
|
413
|
+
* field. Saving is the authorization, as on a create.
|
|
425
414
|
*/
|
|
426
415
|
updatePipeline(id: string, params: PipelineWriteRequest): Promise<PipelineMutationResponse>;
|
|
427
416
|
/**
|
|
@@ -585,6 +574,9 @@ export declare class Loop {
|
|
|
585
574
|
* series that silently dropped the runs where the benchmark could not be
|
|
586
575
|
* scored would read as an unbroken line and not be one, so those points come
|
|
587
576
|
* back with `candidate_score` null and `status_reason` saying why.
|
|
577
|
+
*
|
|
578
|
+
* `pipeline` (a pipeline's id) keeps only the points of that pipeline's
|
|
579
|
+
* attempts, so the trend is one pipeline's attempts by attempt number.
|
|
588
580
|
*/
|
|
589
581
|
getBenchmarkHistory(id: string, params?: BenchmarkHistoryParams): Promise<BenchmarkHistoryResponse>;
|
|
590
582
|
/**
|
package/dist/resources/loop.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { ApiError } from '../client.js';
|
|
1
2
|
/**
|
|
2
3
|
* The Conscious Loop -- pipelines that keep improving a model from the
|
|
3
4
|
* conversations you send them.
|
|
@@ -52,12 +53,25 @@
|
|
|
52
53
|
* const p = await client.loop.getPipeline(pipeline.rule_id);
|
|
53
54
|
* ```
|
|
54
55
|
*/
|
|
56
|
+
/**
|
|
57
|
+
* The code feedback is refused with (409, `Retry-After`) when it names a sample
|
|
58
|
+
* id the gateway issued moments ago (`X-Loop-Sample-Id`) and the sample is not
|
|
59
|
+
* recorded yet. Nothing was stored, so sending the same feedback again is safe.
|
|
60
|
+
*/
|
|
61
|
+
const SAMPLE_NOT_YET_RECORDED = 'SAMPLE_NOT_YET_RECORDED';
|
|
62
|
+
/** Feedback is sent at most this many times while its sample is on its way. */
|
|
63
|
+
const FEEDBACK_TRIES = 6;
|
|
64
|
+
/** The wait when the 409 names none, and the longest wait honoured. */
|
|
65
|
+
const FEEDBACK_RETRY_DEFAULT_SECONDS = 2;
|
|
66
|
+
const FEEDBACK_RETRY_MAX_SECONDS = 30;
|
|
55
67
|
export class Loop {
|
|
56
68
|
_http;
|
|
57
69
|
/** @internal */
|
|
58
70
|
constructor(_http) {
|
|
59
71
|
this._http = _http;
|
|
60
72
|
}
|
|
73
|
+
/** @internal How feedback waits between tries; tests replace it so nothing sleeps. */
|
|
74
|
+
_sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
61
75
|
// ── capture ───────────────────────────────────────────────────────────
|
|
62
76
|
/**
|
|
63
77
|
* Send one exchange to be captured.
|
|
@@ -145,9 +159,18 @@ export class Loop {
|
|
|
145
159
|
return this._http.fetchPost('/api/loop/import', params);
|
|
146
160
|
}
|
|
147
161
|
// ── traces ────────────────────────────────────────────────────────────
|
|
148
|
-
/**
|
|
162
|
+
/**
|
|
163
|
+
* List captured conversations, newest first.
|
|
164
|
+
*
|
|
165
|
+
* `q` searches the text: 2 to 200 characters, any case, matched as written
|
|
166
|
+
* in every message, the answer and its tool calls. Samples kept in your own
|
|
167
|
+
* storage cannot be searched; a search answers how many were left out as
|
|
168
|
+
* `unsearched`.
|
|
169
|
+
*/
|
|
149
170
|
async listTraces(params = {}) {
|
|
150
171
|
const q = new URLSearchParams();
|
|
172
|
+
if (params.q)
|
|
173
|
+
q.set('q', params.q);
|
|
151
174
|
if (params.deployment_id)
|
|
152
175
|
q.set('deployment_id', params.deployment_id);
|
|
153
176
|
if (params.conversation_id)
|
|
@@ -226,10 +249,29 @@ export class Loop {
|
|
|
226
249
|
* `edited` or `gold` verdict is the example, in place of the model's answer;
|
|
227
250
|
* a `rejected` answer is left out. Nothing is learned from an answer that was
|
|
228
251
|
* rejected or replaced. `source` defaults to `human`.
|
|
252
|
+
*
|
|
253
|
+
* Feedback on an `X-Loop-Sample-Id` sent right after the answer can reach
|
|
254
|
+
* the service before the sample does, which answers 409
|
|
255
|
+
* `SAMPLE_NOT_YET_RECORDED` and stores nothing. This method, and so `upvote`,
|
|
256
|
+
* `downvote`, `correct` and `gold`, sends it again after the `Retry-After`
|
|
257
|
+
* seconds (2 when none is given), at most 6 times in all, and then throws
|
|
258
|
+
* that error. No other error is retried.
|
|
229
259
|
*/
|
|
230
260
|
async signal(traceId, params) {
|
|
231
|
-
const
|
|
232
|
-
|
|
261
|
+
const path = `/api/loop/traces/${encodeURIComponent(traceId)}/signals`;
|
|
262
|
+
for (let tries = 1;; tries++) {
|
|
263
|
+
try {
|
|
264
|
+
const res = await this._http.fetchPost(path, params);
|
|
265
|
+
return res.signal;
|
|
266
|
+
}
|
|
267
|
+
catch (err) {
|
|
268
|
+
const onItsWay = err instanceof ApiError && err.status === 409 && err.code === SAMPLE_NOT_YET_RECORDED;
|
|
269
|
+
if (!onItsWay || tries >= FEEDBACK_TRIES)
|
|
270
|
+
throw err;
|
|
271
|
+
const seconds = Math.min(err.retryAfter ?? FEEDBACK_RETRY_DEFAULT_SECONDS, FEEDBACK_RETRY_MAX_SECONDS);
|
|
272
|
+
await this._sleep(seconds * 1000);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
233
275
|
}
|
|
234
276
|
/**
|
|
235
277
|
* Thumbs up: this answer was good. Shorthand for `signal(id, { verdict:
|
|
@@ -481,20 +523,32 @@ export class Loop {
|
|
|
481
523
|
}
|
|
482
524
|
// ── pipelines ─────────────────────────────────────────────────────────
|
|
483
525
|
/**
|
|
484
|
-
*
|
|
485
|
-
* the live model),
|
|
486
|
-
*
|
|
526
|
+
* The workspace's pipelines, a page at a time: its versions (the attempts
|
|
527
|
+
* that became the live model), its latest 5 attempts (`attempts_total` says
|
|
528
|
+
* how many; {@link listPipelineAttempts} pages through every one), which
|
|
529
|
+
* version is live, the attempt in flight, and what the pipeline is waiting
|
|
530
|
+
* for.
|
|
531
|
+
*
|
|
532
|
+
* `limit` is 100 when absent and at most 100; `q` matches the name, the tag
|
|
533
|
+
* or the model (case-insensitive); `status` keeps the pipelines `training`,
|
|
534
|
+
* waiting on you (`needs_you`), `serving` or `paused`.
|
|
487
535
|
*
|
|
488
536
|
* RETURNED WITH ITS ENVELOPE, because the list is not always all of them:
|
|
489
|
-
* the service returns
|
|
490
|
-
*
|
|
491
|
-
*
|
|
492
|
-
*
|
|
537
|
+
* the service returns one page, newest first, `total` is every pipeline
|
|
538
|
+
* that matches and `truncated` is true when more exist after this page. A
|
|
539
|
+
* caller that shows `pipelines` alone presents a short list as the whole of
|
|
540
|
+
* it; the next page is `{ offset: offset + pipelines.length }`.
|
|
493
541
|
*/
|
|
494
542
|
async listPipelines(params = {}) {
|
|
495
543
|
const q = new URLSearchParams();
|
|
544
|
+
if (params.limit != null)
|
|
545
|
+
q.set('limit', String(params.limit));
|
|
496
546
|
if (params.offset)
|
|
497
547
|
q.set('offset', String(params.offset));
|
|
548
|
+
if (params.q)
|
|
549
|
+
q.set('q', params.q);
|
|
550
|
+
if (params.status)
|
|
551
|
+
q.set('status', params.status);
|
|
498
552
|
const qs = q.toString();
|
|
499
553
|
const res = await this._http.fetchGet(`/api/loop/pipelines${qs ? `?${qs}` : ''}`);
|
|
500
554
|
const pipelines = res.pipelines || [];
|
|
@@ -511,14 +565,38 @@ export class Loop {
|
|
|
511
565
|
* `status` says whether it is the platform working (`running`), the member
|
|
512
566
|
* who has to act (`needs_review`, `needs_funds`), or nothing at all until
|
|
513
567
|
* somebody does (`paused`, with `paused_reason` saying why; `complete` at
|
|
514
|
-
* `max_versions`).
|
|
515
|
-
*
|
|
516
|
-
* at most, not an exact spend.
|
|
568
|
+
* `max_versions`). The monthly safety limit is the platform's and has no
|
|
569
|
+
* field: when it holds the pipeline, `next_reason` says so.
|
|
517
570
|
*/
|
|
518
571
|
async getPipeline(id) {
|
|
519
572
|
const res = await this._http.fetchGet(`/api/loop/pipelines/${encodeURIComponent(id)}`);
|
|
520
573
|
return res.pipeline;
|
|
521
574
|
}
|
|
575
|
+
/**
|
|
576
|
+
* A pipeline's attempts, newest first, a page at a time: every training run
|
|
577
|
+
* it made, winner or not, each the shape `attempts` has on the pipeline
|
|
578
|
+
* (with its `explanation` in plain words). `limit` is 50 when absent and at
|
|
579
|
+
* most 200. Narrow by `result` (`became_version`, `not_better`,
|
|
580
|
+
* `inconclusive`, `failed`, `stopped`, `running` -- which includes one
|
|
581
|
+
* waiting for a decision -- or `waiting_for_review`) and by `model` (`main`,
|
|
582
|
+
* or `second` for the second model's attempts); `total` follows the
|
|
583
|
+
* filters.
|
|
584
|
+
*/
|
|
585
|
+
async listPipelineAttempts(id, params = {}) {
|
|
586
|
+
const q = new URLSearchParams();
|
|
587
|
+
if (params.limit != null)
|
|
588
|
+
q.set('limit', String(params.limit));
|
|
589
|
+
if (params.offset)
|
|
590
|
+
q.set('offset', String(params.offset));
|
|
591
|
+
if (params.result)
|
|
592
|
+
q.set('result', params.result);
|
|
593
|
+
if (params.model)
|
|
594
|
+
q.set('model', params.model);
|
|
595
|
+
const qs = q.toString();
|
|
596
|
+
const res = await this._http.fetchGet(`/api/loop/pipelines/${encodeURIComponent(id)}/attempts${qs ? `?${qs}` : ''}`);
|
|
597
|
+
const attempts = res.attempts || [];
|
|
598
|
+
return { attempts, total: typeof res.total === 'number' ? res.total : attempts.length };
|
|
599
|
+
}
|
|
522
600
|
// ── creating and changing pipelines ───────────────────────────────────
|
|
523
601
|
//
|
|
524
602
|
// One body -- a name, a model, tags, when to train, how to promote -- and
|
|
@@ -538,15 +616,18 @@ export class Loop {
|
|
|
538
616
|
* judged on at most `scored_rows` of them). They are read-only.
|
|
539
617
|
* Where a pipeline builds on its live version (its `chain` is set), each
|
|
540
618
|
* attempt trains a new adapter on top of the live version, on what it has
|
|
541
|
-
* not learned yet plus a replay of what it already learned (about
|
|
619
|
+
* not learned yet plus a replay of what it already learned (about half
|
|
542
620
|
* of what it trains on, picked at random, so it is not forgotten), and a
|
|
543
621
|
* winner is merged in and served as one model.
|
|
622
|
+
* `unavailable` lists every other model the platform knows, each with
|
|
623
|
+
* why it is not offered here (`reason_code`, and `reason`, one plain
|
|
624
|
+
* sentence); `[]` from a service from before it was sent.
|
|
544
625
|
* When the list cannot be read it throws `ApiError` 503
|
|
545
626
|
* `MODELS_UNAVAILABLE`, whose `body.floors` still carries the floors.
|
|
546
627
|
*/
|
|
547
628
|
async listModels() {
|
|
548
629
|
const res = await this._http.fetchGet('/api/loop/models');
|
|
549
|
-
return { ...res, models: res.models || [] };
|
|
630
|
+
return { ...res, models: res.models || [], unavailable: res.unavailable || [] };
|
|
550
631
|
}
|
|
551
632
|
/**
|
|
552
633
|
* Create a pipeline -- one use case.
|
|
@@ -558,12 +639,14 @@ export class Loop {
|
|
|
558
639
|
* Where the pipeline builds on its live version (`chain`), each attempt
|
|
559
640
|
* waits for 1,000 new samples and trains a new adapter on top of the live
|
|
560
641
|
* version, on what it has not learned plus a replay of what it already
|
|
561
|
-
* learned (about
|
|
642
|
+
* learned (about half of what it trains on, picked at random); a winner
|
|
562
643
|
* is merged in.
|
|
563
644
|
* What is charged is the GPU time the runs actually use; the platform's own
|
|
564
645
|
* per-run and monthly limits stop a run that goes wrong. `holdout.percent`
|
|
565
646
|
* (5 to 20, default 5) is the share of its samples held back to judge each
|
|
566
|
-
* attempt and never trained on.
|
|
647
|
+
* attempt and never trained on. `training` sets any of its training
|
|
648
|
+
* settings (`epochs`, `learning_rate`, `lora_rank`, `lora_alpha`,
|
|
649
|
+
* `replay_percent`); left out, or null, every one is automatic.
|
|
567
650
|
*/
|
|
568
651
|
async createPipeline(params) {
|
|
569
652
|
return this._http.fetchPost('/api/loop/pipelines', params);
|
|
@@ -573,7 +656,13 @@ export class Loop {
|
|
|
573
656
|
* the limit and `challenger_model: null` removes the challenger. `promotion`
|
|
574
657
|
* is how the promotion policy is set. A new `holdout.percent` is a new
|
|
575
658
|
* revision: lowering it keeps every sample already held back, and raising it
|
|
576
|
-
* holds back only new samples.
|
|
659
|
+
* holds back only new samples. `training` changes only the settings it
|
|
660
|
+
* carries -- `{ epochs: 8 }` sets the passes and keeps the rest,
|
|
661
|
+
* `{ epochs: null }` puts the passes back to automatic -- and
|
|
662
|
+
* `training: null` puts every one back to automatic; the next attempt
|
|
663
|
+
* trains with them, and earlier attempts keep the settings they trained
|
|
664
|
+
* with. Out of range is 422 `TRAINING_SETTING_OUT_OF_RANGE`, naming the
|
|
665
|
+
* field. Saving is the authorization, as on a create.
|
|
577
666
|
*/
|
|
578
667
|
async updatePipeline(id, params) {
|
|
579
668
|
return this._http.fetchPut(`/api/loop/pipelines/${encodeURIComponent(id)}`, params);
|
|
@@ -811,9 +900,14 @@ export class Loop {
|
|
|
811
900
|
* series that silently dropped the runs where the benchmark could not be
|
|
812
901
|
* scored would read as an unbroken line and not be one, so those points come
|
|
813
902
|
* back with `candidate_score` null and `status_reason` saying why.
|
|
903
|
+
*
|
|
904
|
+
* `pipeline` (a pipeline's id) keeps only the points of that pipeline's
|
|
905
|
+
* attempts, so the trend is one pipeline's attempts by attempt number.
|
|
814
906
|
*/
|
|
815
907
|
async getBenchmarkHistory(id, params = {}) {
|
|
816
908
|
const q = new URLSearchParams();
|
|
909
|
+
if (params.pipeline)
|
|
910
|
+
q.set('pipeline', params.pipeline);
|
|
817
911
|
if (params.limit != null)
|
|
818
912
|
q.set('limit', String(params.limit));
|
|
819
913
|
if (params.offset != null)
|
|
@@ -9,8 +9,9 @@ function trainingIdempotencyKey(explicit) {
|
|
|
9
9
|
}
|
|
10
10
|
/**
|
|
11
11
|
* `start_from` on the wire. The adapters stay in the order given (oldest
|
|
12
|
-
* first: the order they are merged in)
|
|
13
|
-
*
|
|
12
|
+
* first: the order they are merged in). An empty list, an `exportStep` other
|
|
13
|
+
* than 'best' or 'final', and an `exportStep` without `exportMerged: true`
|
|
14
|
+
* are refused here rather than sent, since the API refuses each of them too.
|
|
14
15
|
*/
|
|
15
16
|
function startFromBody(sf) {
|
|
16
17
|
if (!Array.isArray(sf.adapters) || sf.adapters.length === 0) {
|
|
@@ -21,6 +22,15 @@ function startFromBody(sf) {
|
|
|
21
22
|
};
|
|
22
23
|
if (sf.exportMerged !== undefined)
|
|
23
24
|
body.export_merged = sf.exportMerged;
|
|
25
|
+
if (sf.exportStep !== undefined) {
|
|
26
|
+
if (sf.exportStep !== 'best' && sf.exportStep !== 'final') {
|
|
27
|
+
throw new Error("RunBiOS: startFrom.exportStep must be 'best' (the step validation picked) or 'final' (the last step trained)");
|
|
28
|
+
}
|
|
29
|
+
if (sf.exportMerged !== true) {
|
|
30
|
+
throw new Error('RunBiOS: startFrom.exportStep names the step the merged model is made of, so it needs exportMerged: true');
|
|
31
|
+
}
|
|
32
|
+
body.export_step = sf.exportStep;
|
|
33
|
+
}
|
|
24
34
|
return body;
|
|
25
35
|
}
|
|
26
36
|
function buildTrainingRequest(params) {
|