runbios-sdk 0.2.1-dev.97 → 0.2.1-rc.118

package/dist/resources/loop.d.ts

@@ -0,0 +1,378 @@
+import type { HttpClient } from '../client.js';
+import type { LoopTrace, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopImportParams, LoopImportResult, LoopSignal, LoopSignalParams, LoopCandidate, LoopCandidateParams, LoopGrader, LoopGraderParams, LoopLabel, LoopLabelCount, LoopGradeResult, LoopDataset, LoopDatasetListParams, LoopDatasetItem, LoopDatasetCreateParams, LoopConfig, LoopConfigParams, LoopStats, LoopJudge, LoopJudgeParams, LoopJudgeRun, LoopJudgeWork, LoopJudgeVerdict, LoopJudgeVerdictResult } from '../types.js';
+/**
+ * The Conscious Loop -- capture what your model was asked and answered, record
+ * whether it was right, and turn those judgements into training data.
+ *
+ * Nothing is captured until you turn it on for a source, and you can turn it
+ * off again at any time. Anything that looks like a credential or a personal
+ * detail is removed before the record is written, never afterwards.
+ *
+ * Requires an API key carrying `loop:read` and/or `loop:write`. Those scopes are
+ * deliberately absent from the Read Only preset, because what they return is
+ * your raw prompts and completions rather than catalog metadata.
+ *
+ * @example
+ * ```ts
+ * // 1. Decide that this source is recorded.
+ * await client.loop.setConfig('my-agent', { enabled: true, retention_days: 30 });
+ *
+ * // 2. Send what your model was asked and what it answered. Works whoever
+ * //    served it -- us, another provider, or your own servers.
+ * const { trace_id } = await client.loop.capture({
+ *   deployment_id: 'my-agent',
+ *   model: 'gpt-4o',
+ *   messages: [{ role: 'user', content: 'what is our refund window' }],
+ *   completion: 'thirty days',
+ * });
+ *
+ * // 3. Say whether it was right. A rewrite is the most valuable answer here:
+ * //    the model learns your version AND learns to avoid its own.
+ * await client.loop.signal(trace_id, {
+ *   verdict: 'edited',
+ *   correction: 'Thirty days from delivery, no questions asked.',
+ * });
+ *
+ * // 4. Turn the judgements into a training set and take the file.
+ * const ds = await client.loop.createDataset({ name: 'support', method: 'sft' });
+ * const jsonl = await client.loop.downloadDataset(ds.id);
+ * ```
+ */
+export declare class Loop {
+    private readonly _http;
+    /** @internal */
+    constructor(_http: HttpClient);
+    /**
+     * Send one exchange to be captured.
+     *
+     * Works regardless of who served the model: ours, OpenAI, Fireworks, your own
+     * own servers, an agent framework. Pass `tool_calls` and `tools` when the turn
+     * invoked a function -- without them a tool-using exchange trains the model to
+     * answer in prose where it should have called something.
+     *
+     * The returned `trace_id` is OURS. If you pass your own `request_id` it is
+     * kept as an idempotency handle: sending the same one again returns the same
+     * trace rather than storing a second copy, so a retry is safe.
+     *
+     * When the source is not enabled this resolves with `captured: false` and a
+     * reason instead of throwing -- capture must never be the thing that breaks
+     * your application.
+     */
+    capture(params: LoopCaptureParams): Promise<LoopCaptureResult>;
+    /**
+     * Bring data you already have into the loop.
+     *
+     * This does NOT create a training set. It creates conversations, in the same
+     * place captured ones live and subject to the same review, rules, judges and
+     * labels. A row becomes trainable when something says it is good, never
+     * because it arrived in a file -- which is the one guarantee that separates
+     * a corpus from a pile.
+     *
+     * Each row is read for what it actually is. A `prompt`/`chosen`/`rejected`
+     * triple becomes a preference pair; an answer plus a yes-or-no becomes a
+     * thumbs verdict; a question and an answer waits for review; a question with
+     * no answer waits for an answer; a paragraph of prose is refused, because it
+     * is not a conversation.
+     *
+     * A verdict that arrives *with* the file is kept -- discarding somebody's
+     * judgement would be worse -- but it is recorded as having come from your
+     * earlier process rather than from a reviewer here, and the conversation
+     * records that it was imported. Neither fact can be reconstructed later, so
+     * both are written at the door.
+     *
+     * `source` is required and becomes the id every imported conversation is
+     * filed under: "everything in one bucket called import" is a corpus nobody
+     * can slice afterwards. At most 5000 rows per call, because the call is
+     * synchronous and somebody is waiting on it.
+     */
+    importRows(params: LoopImportParams): Promise<LoopImportResult>;
+    /** List captured conversations, newest first. */
+    listTraces(params?: LoopTraceListParams): Promise<LoopTraceListResponse>;
+    /** Read one conversation, with every verdict recorded on it. */
+    getTrace(id: string): Promise<LoopTrace>;
+    /**
+     * Delete a conversation.
+     *
+     * It also leaves every training set built from it, including sets already
+     * exported -- so a customer asking you to delete a conversation genuinely
+     * removes it from what the model will learn next. Training runs that have
+     * already finished are not affected.
+     */
+    deleteTrace(id: string): Promise<{
+        deleted: boolean;
+        trace_id: string;
+    }>;
+    /**
+     * Record a verdict on a conversation.
+     *
+     * Append-only: posting a second verdict does not replace the first. Two
+     * reviewers disagreeing about an answer is information worth keeping.
+     *
+     * `source` says who judged: `human`, `verifier` (a deterministic check --
+     * tests passed, schema valid), `judge` (a model grading a model), or
+     * `behavioural` (what the user did next). When several disagree, a human
+     * outranks a verifier, a verifier outranks a judge, and a judge outranks a
+     * behavioural hint.
+     */
+    signal(traceId: string, params: LoopSignalParams): Promise<LoopSignal>;
+    /**
+     * Thumbs up: this answer was good. Shorthand for `signal(id, { verdict:
+     * 'accepted' })` — there is one stored vocabulary behind both, so a thumbs-up
+     * and an "accepted" can never disagree in your corpus.
+     */
+    upvote(traceId: string, reason?: string): Promise<LoopSignal>;
+    /**
+     * Thumbs down: this answer was bad, and you have nothing better to offer.
+     *
+     * On its own this REMOVES the answer from training rather than teaching
+     * anything — there is no better version to learn. If you know what it should
+     * have said, `correct()` is worth far more.
+     */
+    downvote(traceId: string, reason?: string): Promise<LoopSignal>;
+    /**
+     * The model was wrong; here is what it should have said.
+     *
+     * The most valuable feedback there is, because it produces BOTH halves of a
+     * preference pair from one action: the model learns your answer and learns to
+     * avoid its own.
+     */
+    correct(traceId: string, answer: string, reason?: string): Promise<LoopSignal>;
+    /**
+     * Record the GOLD answer for this question — the reference, regardless of
+     * what the model happened to say.
+     *
+     * Different from `correct()` in a way that matters: a correction asserts the
+     * model was wrong, so the original becomes the rejected half of a pair. A gold
+     * answer asserts nothing about the model, so if it MATCHES what was said, no
+     * preference pair is invented — but SFT still learns it.
+     *
+     * It is the most reusable thing you can record: it trains SFT, forms a DPO
+     * pair when it differs from the answer, and stands in as the GRPO reference
+     * when you have not supplied a separate ground truth.
+     */
+    gold(traceId: string, answer: string, reason?: string): Promise<LoopSignal>;
+    /** Every verdict on one conversation, oldest first. */
+    listSignals(traceId: string): Promise<LoopSignal[]>;
+    /**
+     * Submit an alternative answer to a prompt already captured.
+     *
+     * This is what makes preference learning scale. A human rewrite is the best
+     * signal there is and the least available -- somebody has to sit down and
+     * write it. Sample the model several times for the same prompt, score the
+     * samples, and a DPO pair falls out automatically: best becomes chosen,
+     * worst becomes rejected. Point a stronger model at the prompt instead and
+     * the same machinery does distillation.
+     *
+     * Score them, or they cannot pair: an unscored alternative says nothing about
+     * which answer anybody prefers. `score_source` is required alongside a score,
+     * because precedence between a verifier, a judge and a person is the whole
+     * reason we record who judged.
+     *
+     * A human correction always outranks any score.
+     *
+     * @example
+     * ```ts
+     * for (const sample of await sampleMyModel(prompt, 4)) {
+     *   await client.loop.addCandidate(traceId, {
+     *     completion: sample.text,
+     *     score: await myVerifier(sample.text),
+     *     score_source: 'verifier',
+     *   });
+     * }
+     * ```
+     */
+    addCandidate(traceId: string, params: LoopCandidateParams): Promise<LoopCandidate>;
+    /** Every alternative answer recorded for one prompt, oldest first. */
+    listCandidates(traceId: string): Promise<LoopCandidate[]>;
+    /**
+     * Label a conversation, so it can be selected later.
+     *
+     * An average over everything is the least useful thing to train on. A model
+     * weak at refunds is fixed with refund examples, and you can only select
+     * those if you said so when the conversation arrived.
+     *
+     * A label may name a `parent`, which is what makes a MASTER GROUP: labelling
+     * something `refunds` with parent `billing` makes it selectable as either,
+     * and selecting `billing` later gathers every child without you maintaining
+     * a list of them.
+     *
+     * Labels are lowercased and trimmed, so `Refunds` and `refunds` are one label.
+     */
+    label(traceId: string, labels: string[], parent?: string, attributes?: Record<string, string>): Promise<LoopLabel[]>;
+    /**
+     * Set named dimensions on a conversation: `{ category: 'billing',
+     * language: 'es' }`.
+     *
+     * An upsert per dimension, so correcting `category` leaves `task` and any
+     * bare tags exactly where they were — no delete-then-add.
+     */
+    setAttributes(traceId: string, attributes: Record<string, string>): Promise<LoopLabel[]>;
+    unlabel(traceId: string, label: string, key?: string): Promise<{
+        deleted: boolean;
+        key: string;
+        label: string;
+    }>;
+    /** Every label in the workspace, with how many conversations carry it. */
+    listLabels(): Promise<LoopLabelCount[]>;
+    /**
+     * Build a training set from the feedback recorded so far.
+     *
+     * The three methods need genuinely different things, so one set cannot be
+     * reshaped into another afterwards:
+     *
+     * - `sft` -- answers you marked right, and answers you rewrote.
+     * - `dpo` -- answers you REWROTE, so a better and a worse version of the same
+     *   reply exist. Nothing else produces a pair.
+     * - `grpo` -- answers with a value or fact they can be checked against, or
+     *   a judge rubric, which scores answers the run has not written yet.
+     * - `kto` -- anything carrying a yes or a no, INCLUDING a thumbs-down with
+     *   nothing written. That row trains nothing under the other three methods,
+     *   which is why this one exists: it is the feedback people actually give.
+     *
+     * `holdout_percent` holds back your most recent work rather than a random
+     * slice, so the evaluation measures whether the model generalised instead of
+     * memorised the same week.
+     *
+     * The result always reports `rejected_counts`: why rows were left out. A
+     * small set with a reason is useful; a small set without one is just alarming.
+     */
+    createDataset(params: LoopDatasetCreateParams): Promise<LoopDataset>;
+    /** List training sets, newest first. */
+    listDatasets(params?: LoopDatasetListParams): Promise<LoopDataset[]>;
+    /** Read one training set and its curation report. */
+    getDataset(id: string): Promise<LoopDataset>;
+    /**
+     * Page through the rows a training set actually contains.
+     *
+     * Worth reading before you spend money on a run: each row carries the address
+     * of the conversation it was built from.
+     */
+    listDatasetItems(id: string, params?: {
+        limit?: number;
+        offset?: number;
+    }): Promise<LoopDatasetItem[]>;
+    /**
+     * Download a training set as JSONL -- one training row per line, the format
+     * every trainer in this space reads.
+     *
+     * `split` defaults to the training rows; pass `holdout` for the slice held
+     * back, or `all` for both.
+     */
+    downloadDataset(id: string, split?: 'train' | 'holdout' | 'all'): Promise<string>;
+    /**
+     * Delete a training set.
+     *
+     * The conversations it was built from are untouched -- a set is a selection,
+     * and discarding the selection must not discard the evidence.
+     */
+    deleteDataset(id: string): Promise<{
+        deleted: boolean;
+        dataset_id: string;
+    }>;
+    /**
+     * Write a rule that scores answers without a person.
+     *
+     * Human review is the most trustworthy feedback and the least available. A
+     * grader is written once and applied to every answer afterwards: did it
+     * contain the required phrase, did it parse as the schema you asked for, is
+     * the number within tolerance of the known answer, did it call the function
+     * it should have.
+     *
+     * Every kind here is DETERMINISTIC -- no model call, no network. That is why
+     * a verifier outranks a judge when they disagree: it cannot be flattered and
+     * it cannot drift between runs.
+     *
+     * `weight` is the multiplier: a criterion that matters twice as much gets
+     * twice the weight, and the combined score is the weighted mean over the
+     * rules that actually applied.
+     *
+     * A caution worth knowing before you write a set: a rule made only of
+     * `forbidden` phrases is satisfied VACUOUSLY by an answer that says nothing.
+     * Pair it with a `required` phrase, or you are rewarding silence.
+     */
+    createGrader(params: LoopGraderParams): Promise<LoopGrader>;
+    /** Every rule this workspace has written. */
+    listGraders(): Promise<LoopGrader[]>;
+    deleteGrader(id: string): Promise<{
+        deleted: boolean;
+        grader_id: string;
+    }>;
+    /**
+     * Apply this workspace's rules to one captured answer AND to every
+     * alternative sampled for it.
+     *
+     * Grading both in one pass is the point: scoring only the original gives a
+     * verdict, while scoring the samples as well gives the preference pair. Sample
+     * your model, call this, and you have DPO data with nobody reading anything.
+     *
+     * A run where no rule could apply writes NOTHING -- no verdict, no scores.
+     * Recording a zero that no rule produced would poison curation with a
+     * judgement nobody reached, so `applied: 0` is reported instead.
+     */
+    grade(traceId: string): Promise<LoopGradeResult>;
+    /** Every source this workspace has configured. */
+    listConfigs(): Promise<LoopConfig[]>;
+    /**
+     * Read the capture setting for one source. A source nobody has configured
+     * reads back as disabled rather than missing, because "we are not recording
+     * this" is the honest answer.
+     */
+    getConfig(source: string): Promise<LoopConfig>;
+    /**
+     * Turn capture on or off for a source, and choose how long it is kept.
+     *
+     * This is the consent decision the whole feature rests on: nothing is
+     * recorded until it is made, and the row remembers who made it and when.
+     * `sample_rate` below 1 records a deterministic fraction, chosen so that every
+     * turn of one conversation is captured or skipped together -- half a
+     * conversation makes training rows with holes in them.
+     */
+    setConfig(source: string, params: LoopConfigParams): Promise<LoopConfig>;
+    /**
+     * How much there is, and how much of it each method could actually use.
+     *
+     * The `ready` numbers are upper bounds: duplicate questions are folded into
+     * one row while a set is built, so the finished count can be lower.
+     */
+    stats(): Promise<LoopStats>;
+    /** Write a rubric: what makes an answer good here, and what to score. */
+    createJudge(params: LoopJudgeParams): Promise<LoopJudge>;
+    listJudges(): Promise<LoopJudge[]>;
+    /**
+     * Rewrite a rubric. Runs already recorded keep the instructions they used:
+     * a verdict has to keep meaning what it meant when it was given.
+     */
+    updateJudge(judgeId: string, params: LoopJudgeParams): Promise<LoopJudge>;
+    /**
+     * Retire a rubric. Its runs go with it; the verdicts they wrote stay. A score
+     * is evidence about an answer, and it does not stop being true because the
+     * rubric was retired.
+     */
+    deleteJudge(judgeId: string): Promise<void>;
+    /**
+     * Open a run: select the conversations now, and freeze the rubric onto them.
+     *
+     * The selection is fixed at this moment on purpose. A run whose selection is
+     * a live query silently grows as conversations arrive, so "we scored the
+     * refunds slice" becomes a claim about a set that no longer exists.
+     */
+    startRun(judgeId: string, limit?: number): Promise<LoopJudgeRun>;
+    listRuns(judgeId?: string): Promise<LoopJudgeRun[]>;
+    getRun(runId: string): Promise<LoopJudgeRun>;
+    /**
+     * Take the next conversations to score, each rendered into a ready-to-send
+     * prompt. Nothing is marked taken, so a caller that dies half way loses
+     * nothing: ask again and the same items come back.
+     */
+    takeWork(runId: string, limit?: number): Promise<LoopJudgeWork>;
+    /**
+     * Hand the scores back. A verdict for a conversation outside this run, or
+     * scoring something the rubric never asked for, is refused and reported in
+     * `rejected` rather than silently dropped.
+     *
+     * Pass `finish` to close a run you have decided to stop early. Without it an
+     * unfinished run stays open, which is the honest state for work that was
+     * abandoned rather than completed.
+     */
+    postVerdicts(runId: string, verdicts: LoopJudgeVerdict[], finish?: boolean): Promise<LoopJudgeVerdictResult>;
+}