runbios-sdk 0.3.0 → 0.4.0-dev.333

package/dist/resources/loop.js

@@ -138,7 +138,9 @@ export class Loop {
      * an earlier import already had, and the sentences in `notes` are written
      * from those counts, so the numbers and the prose describe the same call and
      * a re-send cannot report a reviewed conversation as waiting for review.
-     * `refused` is rows whose shape could not be read, which the file fixes;
+     * `refused` is rows whose shape could not be read, which the file fixes,
+     * and `refused_reasons` says why, one plain `sentence` per reason with its
+     * `count` (show those, never the codes);
      * `not_saved` is rows that were understood and then could not be stored,
      * which the file cannot fix.
      *
@@ -173,6 +175,22 @@ export class Loop {
      * in every message, the answer and its tool calls. Samples kept in your own
      * storage cannot be searched; a search answers how many were left out as
      * `unsearched`.
+     *
+     * `signal` (`good`, `bad`, `corrected`), `signalled` and `unsignalled` read
+     * a PERSON's feedback: the judge's score of an unreviewed sample is not
+     * feedback, so a sample it scored stays in `unsignalled`. `judge` splits
+     * those unreviewed samples: `picked` (scored 0.8 or more, trained on),
+     * `below_bar` (not trained on) and `unscored`; the three add up to
+     * `unsignalled`. A row the judge scored carries `judge` (`score`,
+     * `picked`).
+     *
+     * `pipeline` lists what one pipeline trains on (its tags or its datasets,
+     * narrowed by its conditions), each row with `state`: what happens to it in
+     * that pipeline (`held_back`, `learned`, `trained`, `waiting` or
+     * `left_out`, with `reason`, the sentence to show). `dataset` lists what
+     * one saved dataset selects; not with `pipeline`. `attempt` (a run id)
+     * with `split` (`train` or `held_back`) lists the samples one attempt
+     * trained on or held back.
      */
     async listTraces(params = {}) {
         const q = new URLSearchParams();
@@ -204,10 +222,18 @@ export class Loop {
             q.set('unlabelled', 'true');
         if (params.pipeline)
             q.set('pipeline', params.pipeline);
+        if (params.dataset)
+            q.set('dataset', params.dataset);
+        if (params.attempt)
+            q.set('attempt', params.attempt);
+        if (params.split)
+            q.set('split', params.split);
         if (params.signal)
             q.set('signal', params.signal);
         if (params.unsignalled)
             q.set('unsignalled', 'true');
+        if (params.judge)
+            q.set('judge', params.judge);
         if (params.limit != null)
             q.set('limit', String(params.limit));
         if (params.offset != null)
@@ -218,6 +244,12 @@ export class Loop {
     /**
      * Read one conversation, with every verdict and tag recorded on it.
      *
+     * When the judge scored it, `judge` carries the score and whether it was
+     * picked, and `signals` holds the judge's score (source `judge`, verdict
+     * `scored`) with `picked`, its per-aspect `metadata.scores` and its
+     * `reason`; `reason_unavailable` means the reason could not be read from
+     * storage just now. A person's feedback decides over it.
+     *
      * A conversation kept in your own storage carries `payload_stored_externally`.
      * When its text could not be read just now it also carries
      * `payload_unavailable`, with `messages` null and `completion` empty: it is
@@ -367,13 +399,20 @@ export class Loop {
         return res.signals;
     }
     /**
-     * The pipelines one conversation feeds: every live pipeline whose tags
-     * select it, and whether each is switched on.
+     * The pipelines one conversation feeds: every live pipeline whose tags or
+     * datasets select it (and whose conditions keep it), and whether each is
+     * switched on.
      *
      * Being listed means the pipeline would select it, not that it will be
      * trained on -- a set still drops answers it cannot use and duplicates, and
-     * holds some back to judge by; `note` says so. An empty list is a real
-     * answer: the conversation trains nothing until it carries a pipeline's tag.
+     * holds some back to judge by; `note` says so. Each pipeline's `state` says
+     * what happens to it there: `held_back` (to test its attempts),
+     * `learned` (by version `version_no`), `trained` (by attempt `attempt_no`,
+     * which did not go live), `waiting` (for the next attempt) or `left_out`;
+     * `reason` is the sentence to show (for `left_out`, why). An empty list is
+     * a real answer: the conversation
+     * trains nothing until it carries a pipeline's tag or joins one of its
+     * datasets.
      */
     async getTracePipelines(traceId) {
         return this._http.fetchGet(`/api/loop/traces/${encodeURIComponent(traceId)}/pipelines`);
@@ -481,6 +520,12 @@ export class Loop {
      * every call was recorded. Rows are per tag: a call naming two tags is
      * counted under each, so never add them up as calls. One pipeline's share
      * is `capture` on its detail.
+     *
+     * The workspace's totals: `samples_recent` (recorded in the last 48
+     * hours), `feedback` (every sample by the feedback a person gave it: good,
+     * bad, corrected or unreviewed; they add up to `traces`) and `pipelines`
+     * (how many there are, and how many are training, waiting on you
+     * (`needs_you`), serving or paused).
      */
     async stats() {
         const res = await this._http.fetchGet('/api/loop/stats');
@@ -491,8 +536,10 @@ export class Loop {
      * its attempt number, the version it became (null when it did not), its
      * base model, its win rate, what it cost -- and each day's feedback counted
      * by verdict. `rule_id` narrows the attempts to one pipeline; the feedback
-     * counts are the workspace's. `truncated` says older attempts exist beyond
-     * `limit`.
+     * counts are the workspace's. `samples` is the samples recorded each day
+     * over `days`, by their feedback today (`good`, `bad`, `corrected`,
+     * `unreviewed`): with `rule_id`, that pipeline's. `truncated` says older
+     * attempts exist beyond `limit`.
      */
     async getMetricsHistory(params = {}) {
         const q = new URLSearchParams();
@@ -509,13 +556,20 @@ export class Loop {
     //
     // Every time a pipeline trains it makes an attempt (a training run). The
     // attempts that went live became its versions.
-    /** Attempts, newest first. Filter by pipeline (`rule_id`) or by the state they are sitting in. */
+    /**
+     * Every pipeline's attempts, newest first. Filter by pipeline (`rule_id`),
+     * by the state they are sitting in, or by what became of them (`result`,
+     * as {@link listPipelineAttempts} takes it). Each row carries `attempt`:
+     * the attempt as its pipeline's own list says it, `explanation` included.
+     */
     async listTrainingRuns(params = {}) {
         const q = new URLSearchParams();
         if (params.rule_id)
             q.set('rule_id', params.rule_id);
         if (params.state)
             q.set('state', params.state);
+        if (params.result)
+            q.set('result', params.result);
         if (params.limit != null)
             q.set('limit', String(params.limit));
         if (params.offset != null)
@@ -530,7 +584,11 @@ export class Loop {
      * `timeline` is written in the same transaction as each state change, so it
      * is the record of what actually happened rather than a reconstruction.
      * `available_actions` is the honest answer to "can I promote this": a button
-     * that cannot work should never be offered.
+     * that cannot work should never be offered. `explanation` is what became
+     * of it in plain words, as on the pipeline's attempts; `run.trained_on` is
+     * what it trained on, as it was then (its tags, conditions and each
+     * dataset at its revision); `traces_filter_attempt` true means
+     * `listTraces({ attempt: id, split })` lists its samples.
      */
     async getTrainingRun(id) {
         return this._http.fetchGet(`/api/loop/training-runs/${encodeURIComponent(id)}`);
@@ -582,8 +640,9 @@ export class Loop {
      * The workspace's pipelines, a page at a time: its versions (the attempts
      * that became the live model), its latest 5 attempts (`attempts_total` says
      * how many; {@link listPipelineAttempts} pages through every one), which
-     * version is live, the attempt in flight, and what the pipeline is waiting
-     * for.
+     * version is live, the attempt in flight, what the pipeline is waiting
+     * for (`minimums`), and how many samples it has recorded (`samples_total`,
+     * `samples_recent` in the last 48 hours).
      *
      * `limit` is 100 when absent and at most 100; `q` matches the name, the tag
      * or the model (case-insensitive); `status` keeps the pipelines `training`,
@@ -623,6 +682,12 @@ export class Loop {
      * somebody does (`paused`, with `paused_reason` saying why; `complete` at
      * `max_versions`). The monthly safety limit is the platform's and has no
      * field: when it holds the pipeline, `next_reason` says so.
+     *
+     * What it trains on: its `tags`, the `datasets` it adds, and the
+     * `conditions` that narrow both. `auto_scoring` is the judge's scoring of
+     * its unreviewed samples (null where the platform does not score them):
+     * `message` is the service's sentence for it, and `minimums.picked_by_judge`
+     * how many of the samples it has the judge picked.
      */
     async getPipeline(id) {
         const res = await this._http.fetchGet(`/api/loop/pipelines/${encodeURIComponent(id)}`);
@@ -633,7 +698,8 @@ export class Loop {
      * it made, winner or not, each the shape `attempts` has on the pipeline
      * (with its `explanation` in plain words). `limit` is 50 when absent and at
      * most 200. Narrow by `result` (`became_version`, `not_better`,
-     * `inconclusive`, `failed`, `stopped`, `running` -- which includes one
+     * `inconclusive` -- tested, could not decide --, `not_tested` -- ended
+     * before it was tested --, `failed`, `stopped`, `running` -- which includes one
      * waiting for a decision -- or `waiting_for_review`) and by `model` (`main`,
      * or `second` for the second model's attempts); `total` follows the
      * filters.
@@ -658,7 +724,8 @@ export class Loop {
     // One body -- a name, a model, tags, when to train, how to promote -- and
     // the platform decides the rest: the machines and the limits on each
     // attempt. Saving a pipeline IS the authorization to spend; you pay the GPU
-    // time its attempts actually use.
+    // time its attempts actually use and, where the platform scores samples,
+    // the judge's scoring of unreviewed samples.
     /**
      * The base models a pipeline can train: only those the engine trains with
      * LoRA SFT and the platform can serve for the comparison, recommended first.
@@ -697,12 +764,21 @@ export class Loop {
      * version, on what it has not learned plus a replay of what it already
      * learned (about half of what it trains on, picked at random); a winner
      * is merged in.
-     * What is charged is the GPU time the runs actually use; the platform's own
-     * per-run and monthly limits stop a run that goes wrong. `holdout.percent`
+     * What is charged is the GPU time the attempts actually use and, where the
+     * platform scores samples, the judge's scoring of unreviewed samples, shown
+     * on the attempt it was for; the platform's own per-run and monthly limits
+     * stop a run that goes wrong. `holdout.percent`
      * (5 to 20, default 5) is the share of its samples held back to judge each
      * attempt and never trained on. `training` sets any of its training
      * settings (`epochs`, `learning_rate`, `lora_rank`, `lora_alpha`,
      * `replay_percent`); left out, or null, every one is automatic.
+     * `datasets` (ids from {@link listDatasets}) adds their samples to what its
+     * tags select, and `conditions` narrows both. The judge's scoring of
+     * unreviewed samples is on unless `auto_scoring: { enabled: false }`.
+     * Where the platform scores samples (the pipeline's `auto_scoring` is not
+     * null), while the pipeline is short of the samples its next attempt
+     * needs, the judge scores samples nobody reviewed and the ones scoring 0.8
+     * or more are trained on; what it cost shows on the attempt it was for.
      */
     async createPipeline(params) {
         return this._http.fetchPost('/api/loop/pipelines', params);
@@ -718,7 +794,12 @@ export class Loop {
      * `training: null` puts every one back to automatic; the next attempt
      * trains with them, and earlier attempts keep the settings they trained
      * with. Out of range is 422 `TRAINING_SETTING_OUT_OF_RANGE`, naming the
-     * field. Saving is the authorization, as on a create.
+     * field. `datasets` (`[]` takes every one off) and `conditions` (null
+     * clears them) change what it trains on from the next attempt: nothing is
+     * trained again, versions already made keep what they learned, and to make
+     * the live version forget samples, delete them ({@link deleteTrace}).
+     * `auto_scoring: { enabled: false }` stops the judge's scoring of
+     * unreviewed samples. Saving is the authorization, as on a create.
      */
     async updatePipeline(id, params) {
         return this._http.fetchPut(`/api/loop/pipelines/${encodeURIComponent(id)}`, params);
@@ -763,6 +844,76 @@ export class Loop {
     async importIntoPipeline(id, params) {
         return this._http.fetchPost(`/api/loop/pipelines/${encodeURIComponent(id)}/import`, params);
     }
+    // ── datasets ──────────────────────────────────────────────────────────
+    //
+    // A dataset is a saved, named set of conditions -- tags (any or all of
+    // them), feedback, where samples came from, which model answered, when
+    // they were recorded -- and a live selection, not a copy: samples that
+    // match later join it. Every pipeline already trains on its own tag; add a
+    // dataset to a pipeline (`datasets` on create or edit) to train it on more.
+    // One dataset can feed any number of pipelines.
+    /** The workspace's datasets, the most recently changed first: `{ datasets, total, limit, offset }`. */
+    async listDatasets(params = {}) {
+        const q = new URLSearchParams();
+        if (params.limit != null)
+            q.set('limit', String(params.limit));
+        if (params.offset)
+            q.set('offset', String(params.offset));
+        const qs = q.toString();
+        const res = await this._http.fetchGet(`/api/loop/datasets${qs ? `?${qs}` : ''}`);
+        const datasets = res.datasets || [];
+        return { ...res, datasets, total: typeof res.total === 'number' ? res.total : datasets.length };
+    }
+    /**
+     * One dataset: its conditions, how many samples it selects now (`samples`,
+     * the same number `listTraces({ dataset: id })` totals) by feedback, how
+     * many are in a benchmark (never trained on), and the pipelines that train
+     * on it (`used_by`).
+     */
+    async getDataset(id) {
+        return this._http.fetchGet(`/api/loop/datasets/${encodeURIComponent(id)}`);
+    }
+    /**
+     * Save a dataset: a `name` (unique in the workspace, whatever its case:
+     * 409 `DATASET_NAME_TAKEN`) and any of its conditions. Nothing trains on
+     * it until a pipeline adds it.
+     */
+    async createDataset(params) {
+        return this._http.fetchPost('/api/loop/datasets', params);
+    }
+    /**
+     * Change a dataset: only the fields sent change (null clears `origin`,
+     * `model`, `from` or `to`; `[]` clears `tags` or `feedback`). Pass
+     * `expected_revision` to be refused 409 `REVISION_MISMATCH` rather than
+     * overwrite someone else's edit. Every pipeline in `used_by` trains on the
+     * new selection from its next attempt: samples that join count as new
+     * samples, nothing is trained again, and versions already made keep what
+     * they learned.
+     */
+    async updateDataset(id, params) {
+        return this._http.fetchPut(`/api/loop/datasets/${encodeURIComponent(id)}`, params);
+    }
+    /**
+     * Delete a dataset. Its samples stay. Every pipeline that trained on it
+     * (`removed_from`) stops taking its samples from its next attempt; versions
+     * already made keep what they learned. It cannot be undone.
+     */
+    async deleteDataset(id) {
+        const res = await this._http.fetchDelete(`/api/loop/datasets/${encodeURIComponent(id)}`);
+        return { ...res, removed_from: res.removed_from || [] };
+    }
+    /**
+     * Count what a set of conditions selects, before saving anything: the
+     * samples, by feedback, how many are in a benchmark, and the models that
+     * answered them. `datasets` adds saved datasets' samples, as a pipeline
+     * does, so a pipeline's tags, datasets and conditions can be counted in
+     * one call. The counts are before duplicates and over-long conversations
+     * are set aside; a pipeline's own `minimums` are the exact training
+     * numbers. It writes nothing.
+     */
+    async previewDataset(params = {}) {
+        return this._http.fetchPost('/api/loop/datasets/preview', params);
+    }
     // ── a pipeline's benchmarks, and its versions side by side ─────────────
     /** The pipeline's benchmarks in priority order (1 is the primary), the ones it stopped using, and its promotion policy. */
     async listPipelineBenchmarks(id) {
@@ -805,7 +956,7 @@ export class Loop {
      * and `.b` are each attempt's mean and pass rate over those same
      * conversations -- the like-for-like figures; each side's `holdout` is over
      * its own, possibly larger, set. When the two were scored differently
-     * (another scoring model or rubric), `shared_sample.comparable` is false,
+     * (another judge or rubric), `shared_sample.comparable` is false,
      * `why` says so, no direction is given and `filter` is refused (400); a
      * conversation whose right answer changed between them has no direction
      * either. Numbers and ids only: read an answer with `listEvaluationItems`.