runbios-sdk 0.2.22 → 0.3.0-dev.322

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,63 +1,11 @@
1
- import type { HttpClient } from '../client.js';
2
- import type { LoopTrace, LoopTraceDeleteResponse, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopImportParams, LoopImportResult, LoopSignal, LoopSignalParams, LoopLabel, LoopLabelCount, LoopConfig, LoopConfigParams, LoopStats, LoopTracePipelinesResponse, LoopMetricsHistory, LoopMetricsHistoryParams, TrainingRuleDeleteResponse, TrainingRunListParams, TrainingRunListResponse, TrainingRunResponse, TrainingRunActionResponse, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, Evaluation, EvaluationItemListParams, EvaluationItemsResponse, Benchmark, BenchmarkCreateParams, BenchmarkHistoryParams, BenchmarkHistoryResponse, BenchmarkItemListParams, BenchmarkItemsResponse, BenchmarkListParams, BenchmarkListResponse, BenchmarkRetireParams, BenchmarkRun, BenchmarkRunItemListParams, BenchmarkRunItemsResponse, Pipeline, PipelineListParams, PipelineListResponse, PipelineWriteRequest, PipelineMutationResponse, LoopModelsResponse, PipelineImportParams, PipelineBenchmarksResponse, PipelineBenchmarkAttachParams, PipelineBenchmarkMoveParams, VersionCompareResponse, VersionCompareParams, AttemptCompareResponse, AttemptCompareParams } from '../types.js';
3
- /**
4
- * The Conscious Loop -- pipelines that keep improving a model from the
5
- * conversations you send them.
6
- *
7
- * A pipeline is one use case: a model, the tags whose samples train it, when
8
- * to train, and how a new version is promoted. Every time it trains it makes
9
- * an attempt; an attempt that goes live becomes the next version: one that
10
- * beats the live model, one a member approves, or, after a deleted
11
- * conversation, a rebuild without it that is not clearly worse (a `manual`
12
- * pipeline waits for a member even then).
13
- *
14
- * Nothing is captured until a source is switched on, and you can switch it
15
- * off again at any time. Creating a pipeline switches on the source named by
16
- * its own tag (`own_tag`), stamping that tag, and deleting it switches that
17
- * source off again. Anything that looks like a credential or a personal detail
18
- * is removed before the record is written, never afterwards.
19
- *
20
- * Requires an API key carrying `loop:read` and/or `loop:write`. Those scopes are
21
- * deliberately absent from the Read Only preset, because what they return is
22
- * your raw prompts and completions rather than catalog metadata.
23
- *
24
- * Offered on the development environment only for now. Anywhere else every
25
- * method here, reads included, rejects with a `ComingSoonError` whose
26
- * `code` is `LOOP_COMING_SOON` (HTTP 403): a deliberate product state, not an
27
- * outage, so retrying cannot succeed there.
28
- *
29
- * @example
30
- * ```ts
31
- * // 1. A pipeline: a model it can train, and its own tag.
32
- * const { models } = await client.loop.listModels();
33
- * const { pipeline } = await client.loop.createPipeline({
34
- * name: 'Support replies',
35
- * model: { id: models[0].id },
36
- * });
37
- *
38
- * // 2. Send samples from your app to the pipeline's own source: its create
39
- * // switched that source on, and it tags everything it records.
40
- * const sent = await client.loop.capture({
41
- * deployment_id: pipeline.own_tag!,
42
- * model: 'gpt-4o',
43
- * messages: [{ role: 'user', content: 'what is our refund window' }],
44
- * completion: 'thirty days',
45
- * });
46
- *
47
- * // 3. Say whether it was right, on the trace_id the capture answered with.
48
- * // A correction is the most valuable answer.
49
- * if (sent.trace_id) {
50
- * await client.loop.correct(sent.trace_id, 'Thirty days from delivery, no questions asked.');
51
- * }
52
- *
53
- * // 4. Once it has its minimum, it trains; versions appear on the pipeline.
54
- * const p = await client.loop.getPipeline(pipeline.rule_id);
55
- * ```
56
- */
1
+ import { type HttpClient } from '../client.js';
2
+ import type { LoopTrace, LoopTraceDeleteResponse, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopImportParams, LoopImportResult, LoopSignal, LoopSignalParams, LoopLabel, LoopLabelCount, LoopConfig, LoopConfigParams, LoopStats, LoopTracePipelinesResponse, LoopMetricsHistory, LoopMetricsHistoryParams, TrainingRuleDeleteResponse, TrainingRunListParams, TrainingRunListResponse, TrainingRunResponse, TrainingRunActionResponse, TrainingRunPromoteRequest, TrainingRunRejectRequest, TrainingRunRollbackRequest, TrainingRunCancelRequest, Evaluation, EvaluationItemListParams, EvaluationItemsResponse, Benchmark, BenchmarkCreateParams, BenchmarkHistoryParams, BenchmarkHistoryResponse, BenchmarkItemListParams, BenchmarkItemsResponse, BenchmarkListParams, BenchmarkListResponse, BenchmarkRetireParams, BenchmarkRun, BenchmarkRunItemListParams, BenchmarkRunItemsResponse, Pipeline, PipelineAttemptListParams, PipelineAttemptListResponse, PipelineListParams, PipelineListResponse, PipelineWriteRequest, PipelineMutationResponse, LoopModelsResponse, PipelineImportParams, PipelineBenchmarksResponse, PipelineBenchmarkAttachParams, PipelineBenchmarkMoveParams, VersionCompareResponse, VersionCompareParams, AttemptCompareResponse, AttemptCompareParams } from '../types.js';
57
3
  export declare class Loop {
58
4
  private readonly _http;
59
5
  /** @internal */
60
6
  constructor(_http: HttpClient);
7
+ /** @internal How feedback waits between tries; tests replace it so nothing sleeps. */
8
+ _sleep: (ms: number) => Promise<void>;
61
9
  /**
62
10
  * Send one exchange to be captured.
63
11
  *
@@ -139,7 +87,14 @@ export declare class Loop {
139
87
  * a shorter list moves every row after the gap and imports it again.
140
88
  */
141
89
  importRows(params: LoopImportParams): Promise<LoopImportResult>;
142
- /** List captured conversations, newest first. */
90
+ /**
91
+ * List captured conversations, newest first.
92
+ *
93
+ * `q` searches the text: 2 to 200 characters, any case, matched as written
94
+ * in every message, the answer and its tool calls. Samples kept in your own
95
+ * storage cannot be searched; a search answers how many were left out as
96
+ * `unsearched`.
97
+ */
143
98
  listTraces(params?: LoopTraceListParams): Promise<LoopTraceListResponse>;
144
99
  /**
145
100
  * Read one conversation, with every verdict and tag recorded on it.
@@ -176,6 +131,13 @@ export declare class Loop {
176
131
  * `edited` or `gold` verdict is the example, in place of the model's answer;
177
132
  * a `rejected` answer is left out. Nothing is learned from an answer that was
178
133
  * rejected or replaced. `source` defaults to `human`.
134
+ *
135
+ * Feedback on an `X-Loop-Sample-Id` sent right after the answer can reach
136
+ * the service before the sample does, which answers 409
137
+ * `SAMPLE_NOT_YET_RECORDED` and stores nothing. This method, and so `upvote`,
138
+ * `downvote`, `correct` and `gold`, sends it again after the `Retry-After`
139
+ * seconds (2 when none is given), at most 6 times in all, and then throws
140
+ * that error. No other error is retried.
179
141
  */
180
142
  signal(traceId: string, params: LoopSignalParams): Promise<LoopSignal>;
181
143
  /**
@@ -356,15 +318,21 @@ export declare class Loop {
356
318
  */
357
319
  cancelTrainingRun(id: string, params?: TrainingRunCancelRequest): Promise<TrainingRunActionResponse>;
358
320
  /**
359
- * Every pipeline in the workspace: its versions (the attempts that became
360
- * the live model), every attempt, which version is live, the attempt in
361
- * flight, and what the pipeline is waiting for.
321
+ * The workspace's pipelines, a page at a time: its versions (the attempts
322
+ * that became the live model), its latest 5 attempts (`attempts_total` says
323
+ * how many; {@link listPipelineAttempts} pages through every one), which
324
+ * version is live, the attempt in flight, and what the pipeline is waiting
325
+ * for.
326
+ *
327
+ * `limit` is 100 when absent and at most 100; `q` matches the name, the tag
328
+ * or the model (case-insensitive); `status` keeps the pipelines `training`,
329
+ * waiting on you (`needs_you`), `serving` or `paused`.
362
330
  *
363
331
  * RETURNED WITH ITS ENVELOPE, because the list is not always all of them:
364
- * the service returns a page of 100, newest first, `total` is every
365
- * pipeline in the workspace and `truncated` is true when more exist after
366
- * this page. A caller that shows `pipelines` alone presents a short list as
367
- * the whole of it; the next page is `{ offset: offset + pipelines.length }`.
332
+ * the service returns one page, newest first, `total` is every pipeline
333
+ * that matches and `truncated` is true when more exist after this page. A
334
+ * caller that shows `pipelines` alone presents a short list as the whole of
335
+ * it; the next page is `{ offset: offset + pipelines.length }`.
368
336
  */
369
337
  listPipelines(params?: PipelineListParams): Promise<PipelineListResponse>;
370
338
  /**
@@ -373,11 +341,21 @@ export declare class Loop {
373
341
  * `status` says whether it is the platform working (`running`), the member
374
342
  * who has to act (`needs_review`, `needs_funds`), or nothing at all until
375
343
  * somebody does (`paused`, with `paused_reason` saying why; `complete` at
376
- * `max_versions`). `month_spent_cents` is the figure the
377
- * monthly limit is enforced against: what runs started this month have cost
378
- * at most, not an exact spend.
344
+ * `max_versions`). The monthly safety limit is the platform's and has no
345
+ * field: when it holds the pipeline, `next_reason` says so.
379
346
  */
380
347
  getPipeline(id: string): Promise<Pipeline>;
348
+ /**
349
+ * A pipeline's attempts, newest first, a page at a time: every training run
350
+ * it made, winner or not, each the shape `attempts` has on the pipeline
351
+ * (with its `explanation` in plain words). `limit` is 50 when absent and at
352
+ * most 200. Narrow by `result` (`became_version`, `not_better`,
353
+ * `inconclusive`, `failed`, `stopped`, `running` -- which includes one
354
+ * waiting for a decision -- or `waiting_for_review`) and by `model` (`main`,
355
+ * or `second` for the second model's attempts); `total` follows the
356
+ * filters.
357
+ */
358
+ listPipelineAttempts(id: string, params?: PipelineAttemptListParams): Promise<PipelineAttemptListResponse>;
381
359
  /**
382
360
  * The base models a pipeline can train: only those the engine trains with
383
361
  * LoRA SFT and the platform can serve for the comparison, recommended first.
@@ -391,9 +369,12 @@ export declare class Loop {
391
369
  * judged on at most `scored_rows` of them). They are read-only.
392
370
  * Where a pipeline builds on its live version (its `chain` is set), each
393
371
  * attempt trains a new adapter on top of the live version, on what it has
394
- * not learned yet plus a replay of what it already learned (about a fifth
372
+ * not learned yet plus a replay of what it already learned (about half
395
373
  * of what it trains on, picked at random, so it is not forgotten), and a
396
374
  * winner is merged in and served as one model.
375
+ * `unavailable` lists every other model the platform knows, each with
376
+ * why it is not offered here (`reason_code`, and `reason`, one plain
377
+ * sentence); `[]` from a service from before it was sent.
397
378
  * When the list cannot be read it throws `ApiError` 503
398
379
  * `MODELS_UNAVAILABLE`, whose `body.floors` still carries the floors.
399
380
  */
@@ -408,12 +389,14 @@ export declare class Loop {
408
389
  * Where the pipeline builds on its live version (`chain`), each attempt
409
390
  * waits for 1,000 new samples and trains a new adapter on top of the live
410
391
  * version, on what it has not learned plus a replay of what it already
411
- * learned (about a fifth of what it trains on, picked at random); a winner
392
+ * learned (about half of what it trains on, picked at random); a winner
412
393
  * is merged in.
413
394
  * What is charged is the GPU time the runs actually use; the platform's own
414
395
  * per-run and monthly limits stop a run that goes wrong. `holdout.percent`
415
396
  * (5 to 20, default 5) is the share of its samples held back to judge each
416
- * attempt and never trained on.
397
+ * attempt and never trained on. `training` sets any of its training
398
+ * settings (`epochs`, `learning_rate`, `lora_rank`, `lora_alpha`,
399
+ * `replay_percent`); left out, or null, every one is automatic.
417
400
  */
418
401
  createPipeline(params: PipelineWriteRequest): Promise<PipelineMutationResponse>;
419
402
  /**
@@ -421,7 +404,13 @@ export declare class Loop {
421
404
  * the limit and `challenger_model: null` removes the challenger. `promotion`
422
405
  * is how the promotion policy is set. A new `holdout.percent` is a new
423
406
  * revision: lowering it keeps every sample already held back, and raising it
424
- * holds back only new samples. Saving is the authorization, as on a create.
407
+ * holds back only new samples. `training` changes only the settings it
408
+ * carries -- `{ epochs: 8 }` sets the passes and keeps the rest,
409
+ * `{ epochs: null }` puts the passes back to automatic -- and
410
+ * `training: null` puts every one back to automatic; the next attempt
411
+ * trains with them, and earlier attempts keep the settings they trained
412
+ * with. Out of range is 422 `TRAINING_SETTING_OUT_OF_RANGE`, naming the
413
+ * field. Saving is the authorization, as on a create.
425
414
  */
426
415
  updatePipeline(id: string, params: PipelineWriteRequest): Promise<PipelineMutationResponse>;
427
416
  /**
@@ -585,6 +574,9 @@ export declare class Loop {
585
574
  * series that silently dropped the runs where the benchmark could not be
586
575
  * scored would read as an unbroken line and not be one, so those points come
587
576
  * back with `candidate_score` null and `status_reason` saying why.
577
+ *
578
+ * `pipeline` (a pipeline's id) keeps only the points of that pipeline's
579
+ * attempts, so the trend is one pipeline's attempts by attempt number.
588
580
  */
589
581
  getBenchmarkHistory(id: string, params?: BenchmarkHistoryParams): Promise<BenchmarkHistoryResponse>;
590
582
  /**
@@ -1,3 +1,4 @@
1
+ import { ApiError } from '../client.js';
1
2
  /**
2
3
  * The Conscious Loop -- pipelines that keep improving a model from the
3
4
  * conversations you send them.
@@ -52,12 +53,25 @@
52
53
  * const p = await client.loop.getPipeline(pipeline.rule_id);
53
54
  * ```
54
55
  */
56
+ /**
57
+ * The code feedback is refused with (409, `Retry-After`) when it names a sample
58
+ * id the gateway issued moments ago (`X-Loop-Sample-Id`) and the sample is not
59
+ * recorded yet. Nothing was stored, so sending the same feedback again is safe.
60
+ */
61
+ const SAMPLE_NOT_YET_RECORDED = 'SAMPLE_NOT_YET_RECORDED';
62
+ /** Feedback is sent at most this many times while its sample is on its way. */
63
+ const FEEDBACK_TRIES = 6;
64
+ /** The wait when the 409 names none, and the longest wait honoured. */
65
+ const FEEDBACK_RETRY_DEFAULT_SECONDS = 2;
66
+ const FEEDBACK_RETRY_MAX_SECONDS = 30;
55
67
  export class Loop {
56
68
  _http;
57
69
  /** @internal */
58
70
  constructor(_http) {
59
71
  this._http = _http;
60
72
  }
73
+ /** @internal How feedback waits between tries; tests replace it so nothing sleeps. */
74
+ _sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
61
75
  // ── capture ───────────────────────────────────────────────────────────
62
76
  /**
63
77
  * Send one exchange to be captured.
@@ -145,9 +159,18 @@ export class Loop {
145
159
  return this._http.fetchPost('/api/loop/import', params);
146
160
  }
147
161
  // ── traces ────────────────────────────────────────────────────────────
148
- /** List captured conversations, newest first. */
162
+ /**
163
+ * List captured conversations, newest first.
164
+ *
165
+ * `q` searches the text: 2 to 200 characters, any case, matched as written
166
+ * in every message, the answer and its tool calls. Samples kept in your own
167
+ * storage cannot be searched; a search answers how many were left out as
168
+ * `unsearched`.
169
+ */
149
170
  async listTraces(params = {}) {
150
171
  const q = new URLSearchParams();
172
+ if (params.q)
173
+ q.set('q', params.q);
151
174
  if (params.deployment_id)
152
175
  q.set('deployment_id', params.deployment_id);
153
176
  if (params.conversation_id)
@@ -226,10 +249,29 @@ export class Loop {
226
249
  * `edited` or `gold` verdict is the example, in place of the model's answer;
227
250
  * a `rejected` answer is left out. Nothing is learned from an answer that was
228
251
  * rejected or replaced. `source` defaults to `human`.
252
+ *
253
+ * Feedback on an `X-Loop-Sample-Id` sent right after the answer can reach
254
+ * the service before the sample does, which answers 409
255
+ * `SAMPLE_NOT_YET_RECORDED` and stores nothing. This method, and so `upvote`,
256
+ * `downvote`, `correct` and `gold`, sends it again after the `Retry-After`
257
+ * seconds (2 when none is given), at most 6 times in all, and then throws
258
+ * that error. No other error is retried.
229
259
  */
230
260
  async signal(traceId, params) {
231
- const res = await this._http.fetchPost(`/api/loop/traces/${encodeURIComponent(traceId)}/signals`, params);
232
- return res.signal;
261
+ const path = `/api/loop/traces/${encodeURIComponent(traceId)}/signals`;
262
+ for (let tries = 1;; tries++) {
263
+ try {
264
+ const res = await this._http.fetchPost(path, params);
265
+ return res.signal;
266
+ }
267
+ catch (err) {
268
+ const onItsWay = err instanceof ApiError && err.status === 409 && err.code === SAMPLE_NOT_YET_RECORDED;
269
+ if (!onItsWay || tries >= FEEDBACK_TRIES)
270
+ throw err;
271
+ const seconds = Math.min(err.retryAfter ?? FEEDBACK_RETRY_DEFAULT_SECONDS, FEEDBACK_RETRY_MAX_SECONDS);
272
+ await this._sleep(seconds * 1000);
273
+ }
274
+ }
233
275
  }
234
276
  /**
235
277
  * Thumbs up: this answer was good. Shorthand for `signal(id, { verdict:
@@ -481,20 +523,32 @@ export class Loop {
481
523
  }
482
524
  // ── pipelines ─────────────────────────────────────────────────────────
483
525
  /**
484
- * Every pipeline in the workspace: its versions (the attempts that became
485
- * the live model), every attempt, which version is live, the attempt in
486
- * flight, and what the pipeline is waiting for.
526
+ * The workspace's pipelines, a page at a time: its versions (the attempts
527
+ * that became the live model), its latest 5 attempts (`attempts_total` says
528
+ * how many; {@link listPipelineAttempts} pages through every one), which
529
+ * version is live, the attempt in flight, and what the pipeline is waiting
530
+ * for.
531
+ *
532
+ * `limit` is 100 when absent and at most 100; `q` matches the name, the tag
533
+ * or the model (case-insensitive); `status` keeps the pipelines `training`,
534
+ * waiting on you (`needs_you`), `serving` or `paused`.
487
535
  *
488
536
  * RETURNED WITH ITS ENVELOPE, because the list is not always all of them:
489
- * the service returns a page of 100, newest first, `total` is every
490
- * pipeline in the workspace and `truncated` is true when more exist after
491
- * this page. A caller that shows `pipelines` alone presents a short list as
492
- * the whole of it; the next page is `{ offset: offset + pipelines.length }`.
537
+ * the service returns one page, newest first, `total` is every pipeline
538
+ * that matches and `truncated` is true when more exist after this page. A
539
+ * caller that shows `pipelines` alone presents a short list as the whole of
540
+ * it; the next page is `{ offset: offset + pipelines.length }`.
493
541
  */
494
542
  async listPipelines(params = {}) {
495
543
  const q = new URLSearchParams();
544
+ if (params.limit != null)
545
+ q.set('limit', String(params.limit));
496
546
  if (params.offset)
497
547
  q.set('offset', String(params.offset));
548
+ if (params.q)
549
+ q.set('q', params.q);
550
+ if (params.status)
551
+ q.set('status', params.status);
498
552
  const qs = q.toString();
499
553
  const res = await this._http.fetchGet(`/api/loop/pipelines${qs ? `?${qs}` : ''}`);
500
554
  const pipelines = res.pipelines || [];
@@ -511,14 +565,38 @@ export class Loop {
511
565
  * `status` says whether it is the platform working (`running`), the member
512
566
  * who has to act (`needs_review`, `needs_funds`), or nothing at all until
513
567
  * somebody does (`paused`, with `paused_reason` saying why; `complete` at
514
- * `max_versions`). `month_spent_cents` is the figure the
515
- * monthly limit is enforced against: what runs started this month have cost
516
- * at most, not an exact spend.
568
+ * `max_versions`). The monthly safety limit is the platform's and has no
569
+ * field: when it holds the pipeline, `next_reason` says so.
517
570
  */
518
571
  async getPipeline(id) {
519
572
  const res = await this._http.fetchGet(`/api/loop/pipelines/${encodeURIComponent(id)}`);
520
573
  return res.pipeline;
521
574
  }
575
+ /**
576
+ * A pipeline's attempts, newest first, a page at a time: every training run
577
+ * it made, winner or not, each the shape `attempts` has on the pipeline
578
+ * (with its `explanation` in plain words). `limit` is 50 when absent and at
579
+ * most 200. Narrow by `result` (`became_version`, `not_better`,
580
+ * `inconclusive`, `failed`, `stopped`, `running` -- which includes one
581
+ * waiting for a decision -- or `waiting_for_review`) and by `model` (`main`,
582
+ * or `second` for the second model's attempts); `total` follows the
583
+ * filters.
584
+ */
585
+ async listPipelineAttempts(id, params = {}) {
586
+ const q = new URLSearchParams();
587
+ if (params.limit != null)
588
+ q.set('limit', String(params.limit));
589
+ if (params.offset)
590
+ q.set('offset', String(params.offset));
591
+ if (params.result)
592
+ q.set('result', params.result);
593
+ if (params.model)
594
+ q.set('model', params.model);
595
+ const qs = q.toString();
596
+ const res = await this._http.fetchGet(`/api/loop/pipelines/${encodeURIComponent(id)}/attempts${qs ? `?${qs}` : ''}`);
597
+ const attempts = res.attempts || [];
598
+ return { attempts, total: typeof res.total === 'number' ? res.total : attempts.length };
599
+ }
522
600
  // ── creating and changing pipelines ───────────────────────────────────
523
601
  //
524
602
  // One body -- a name, a model, tags, when to train, how to promote -- and
@@ -538,15 +616,18 @@ export class Loop {
538
616
  * judged on at most `scored_rows` of them). They are read-only.
539
617
  * Where a pipeline builds on its live version (its `chain` is set), each
540
618
  * attempt trains a new adapter on top of the live version, on what it has
541
- * not learned yet plus a replay of what it already learned (about a fifth
619
+ * not learned yet plus a replay of what it already learned (about half
542
620
  * of what it trains on, picked at random, so it is not forgotten), and a
543
621
  * winner is merged in and served as one model.
622
+ * `unavailable` lists every other model the platform knows, each with
623
+ * why it is not offered here (`reason_code`, and `reason`, one plain
624
+ * sentence); `[]` from a service from before it was sent.
544
625
  * When the list cannot be read it throws `ApiError` 503
545
626
  * `MODELS_UNAVAILABLE`, whose `body.floors` still carries the floors.
546
627
  */
547
628
  async listModels() {
548
629
  const res = await this._http.fetchGet('/api/loop/models');
549
- return { ...res, models: res.models || [] };
630
+ return { ...res, models: res.models || [], unavailable: res.unavailable || [] };
550
631
  }
551
632
  /**
552
633
  * Create a pipeline -- one use case.
@@ -558,12 +639,14 @@ export class Loop {
558
639
  * Where the pipeline builds on its live version (`chain`), each attempt
559
640
  * waits for 1,000 new samples and trains a new adapter on top of the live
560
641
  * version, on what it has not learned plus a replay of what it already
561
- * learned (about a fifth of what it trains on, picked at random); a winner
642
+ * learned (about half of what it trains on, picked at random); a winner
562
643
  * is merged in.
563
644
  * What is charged is the GPU time the runs actually use; the platform's own
564
645
  * per-run and monthly limits stop a run that goes wrong. `holdout.percent`
565
646
  * (5 to 20, default 5) is the share of its samples held back to judge each
566
- * attempt and never trained on.
647
+ * attempt and never trained on. `training` sets any of its training
648
+ * settings (`epochs`, `learning_rate`, `lora_rank`, `lora_alpha`,
649
+ * `replay_percent`); left out, or null, every one is automatic.
567
650
  */
568
651
  async createPipeline(params) {
569
652
  return this._http.fetchPost('/api/loop/pipelines', params);
@@ -573,7 +656,13 @@ export class Loop {
573
656
  * the limit and `challenger_model: null` removes the challenger. `promotion`
574
657
  * is how the promotion policy is set. A new `holdout.percent` is a new
575
658
  * revision: lowering it keeps every sample already held back, and raising it
576
- * holds back only new samples. Saving is the authorization, as on a create.
659
+ * holds back only new samples. `training` changes only the settings it
660
+ * carries -- `{ epochs: 8 }` sets the passes and keeps the rest,
661
+ * `{ epochs: null }` puts the passes back to automatic -- and
662
+ * `training: null` puts every one back to automatic; the next attempt
663
+ * trains with them, and earlier attempts keep the settings they trained
664
+ * with. Out of range is 422 `TRAINING_SETTING_OUT_OF_RANGE`, naming the
665
+ * field. Saving is the authorization, as on a create.
577
666
  */
578
667
  async updatePipeline(id, params) {
579
668
  return this._http.fetchPut(`/api/loop/pipelines/${encodeURIComponent(id)}`, params);
@@ -811,9 +900,14 @@ export class Loop {
811
900
  * series that silently dropped the runs where the benchmark could not be
812
901
  * scored would read as an unbroken line and not be one, so those points come
813
902
  * back with `candidate_score` null and `status_reason` saying why.
903
+ *
904
+ * `pipeline` (a pipeline's id) keeps only the points of that pipeline's
905
+ * attempts, so the trend is one pipeline's attempts by attempt number.
814
906
  */
815
907
  async getBenchmarkHistory(id, params = {}) {
816
908
  const q = new URLSearchParams();
909
+ if (params.pipeline)
910
+ q.set('pipeline', params.pipeline);
817
911
  if (params.limit != null)
818
912
  q.set('limit', String(params.limit));
819
913
  if (params.offset != null)
@@ -9,8 +9,9 @@ function trainingIdempotencyKey(explicit) {
9
9
  }
10
10
  /**
11
11
  * `start_from` on the wire. The adapters stay in the order given (oldest
12
- * first: the order they are merged in), and an empty list is refused here
13
- * rather than sent, since the API refuses it too.
12
+ * first: the order they are merged in). An empty list, an `exportStep` other
13
+ * than 'best' or 'final', and an `exportStep` without `exportMerged: true`
14
+ * are refused here rather than sent, since the API refuses each of them too.
14
15
  */
15
16
  function startFromBody(sf) {
16
17
  if (!Array.isArray(sf.adapters) || sf.adapters.length === 0) {
@@ -21,6 +22,15 @@ function startFromBody(sf) {
21
22
  };
22
23
  if (sf.exportMerged !== undefined)
23
24
  body.export_merged = sf.exportMerged;
25
+ if (sf.exportStep !== undefined) {
26
+ if (sf.exportStep !== 'best' && sf.exportStep !== 'final') {
27
+ throw new Error("RunBiOS: startFrom.exportStep must be 'best' (the step validation picked) or 'final' (the last step trained)");
28
+ }
29
+ if (sf.exportMerged !== true) {
30
+ throw new Error('RunBiOS: startFrom.exportStep names the step the merged model is made of, so it needs exportMerged: true');
31
+ }
32
+ body.export_step = sf.exportStep;
33
+ }
24
34
  return body;
25
35
  }
26
36
  function buildTrainingRequest(params) {