runbios-sdk 0.2.1-dev.109 → 0.2.1-dev.110

package/dist/index.d.ts

@@ -35,7 +35,7 @@ import { Loop } from './resources/loop.js';
  * SDK version. Sent as part of the User-Agent header.
  * Must match package.json "version" -- enforced by a contract test.
  */
-export declare const VERSION = "0.2.1-dev.109";
+export declare const VERSION = "0.2.1-dev.110";
 export declare class RunBiOS {
     /** Search models, fetch configs, check adapter compatibility. */
     readonly models: Models;

package/dist/index.js

@@ -35,7 +35,7 @@ import { Loop } from './resources/loop.js';
  * SDK version. Sent as part of the User-Agent header.
  * Must match package.json "version" -- enforced by a contract test.
  */
-export const VERSION = '0.2.1-dev.109';
+export const VERSION = '0.2.1-dev.110';
 export class RunBiOS {
     /** Search models, fetch configs, check adapter compatibility. */
     models;

package/dist/resources/loop.d.ts

@@ -1,5 +1,5 @@
 import type { HttpClient } from '../client.js';
-import type { LoopTrace, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopSignal, LoopSignalParams, LoopCandidate, LoopCandidateParams, LoopGrader, LoopGraderParams, LoopLabel, LoopLabelCount, LoopGradeResult, LoopDataset, LoopDatasetListParams, LoopDatasetItem, LoopDatasetCreateParams, LoopConfig, LoopConfigParams, LoopStats } from '../types.js';
+import type { LoopTrace, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopSignal, LoopSignalParams, LoopCandidate, LoopCandidateParams, LoopGrader, LoopGraderParams, LoopLabel, LoopLabelCount, LoopGradeResult, LoopDataset, LoopDatasetListParams, LoopDatasetItem, LoopDatasetCreateParams, LoopConfig, LoopConfigParams, LoopStats, LoopJudge, LoopJudgeParams, LoopJudgeRun, LoopJudgeWork, LoopJudgeVerdict, LoopJudgeVerdictResult } from '../types.js';
 /**
  * The Conscious Loop -- capture what your model was asked and answered, record
  * whether it was right, and turn those judgements into training data.
@@ -295,4 +295,44 @@ export declare class Loop {
      * one row while a set is built, so the finished count can be lower.
      */
     stats(): Promise<LoopStats>;
+    /** Write a rubric: what makes an answer good here, and what to score. */
+    createJudge(params: LoopJudgeParams): Promise<LoopJudge>;
+    listJudges(): Promise<LoopJudge[]>;
+    /**
+     * Rewrite a rubric. Runs already recorded keep the instructions they used:
+     * a verdict has to keep meaning what it meant when it was given.
+     */
+    updateJudge(judgeId: string, params: LoopJudgeParams): Promise<LoopJudge>;
+    /**
+     * Retire a rubric. Its runs go with it; the verdicts they wrote stay. A score
+     * is evidence about an answer, and it does not stop being true because the
+     * rubric was retired.
+     */
+    deleteJudge(judgeId: string): Promise<void>;
+    /**
+     * Open a run: select the conversations now, and freeze the rubric onto them.
+     *
+     * The selection is fixed at this moment on purpose. A run whose selection is
+     * a live query silently grows as conversations arrive, so "we scored the
+     * refunds slice" becomes a claim about a set that no longer exists.
+     */
+    startRun(judgeId: string, limit?: number): Promise<LoopJudgeRun>;
+    listRuns(judgeId?: string): Promise<LoopJudgeRun[]>;
+    getRun(runId: string): Promise<LoopJudgeRun>;
+    /**
+     * Take the next conversations to score, each rendered into a ready-to-send
+     * prompt. Nothing is marked taken, so a caller that dies half way loses
+     * nothing: ask again and the same items come back.
+     */
+    takeWork(runId: string, limit?: number): Promise<LoopJudgeWork>;
+    /**
+     * Hand the scores back. A verdict for a conversation outside this run, or
+     * scoring something the rubric never asked for, is refused and reported in
+     * `rejected` rather than silently dropped.
+     *
+     * Pass `finish` to close a run you have decided to stop early. Without it an
+     * unfinished run stays open, which is the honest state for work that was
+     * abandoned rather than completed.
+     */
+    postVerdicts(runId: string, verdicts: LoopJudgeVerdict[], finish?: boolean): Promise<LoopJudgeVerdictResult>;
 }

package/dist/resources/loop.js

@@ -394,4 +394,82 @@ export class Loop {
         const res = await this._http.fetchGet('/api/loop/stats');
         return res.stats;
     }
+    // ── judges ────────────────────────────────────────────────────────────
+    //
+    // A grader is a rule: deterministic, cheap, and blind to anything it was not
+    // told to look for. A judge is a rubric handed to a model, which is the only
+    // thing that can answer "was this answer actually helpful".
+    //
+    // YOU run the model. This service stores raw prompts and completions and
+    // holds them with no outbound credentials at all, which is most of the reason
+    // it is safe to store them there -- so it hands the work out instead. Open a
+    // run, take the batch, send each `prompt` to whatever model you like, and
+    // post the scores back.
+    /** Write a rubric: what makes an answer good here, and what to score. */
+    async createJudge(params) {
+        const res = await this._http.fetchPost('/api/loop/judges', params);
+        return res.judge;
+    }
+    async listJudges() {
+        const res = await this._http.fetchGet('/api/loop/judges');
+        return res.judges;
+    }
+    /**
+     * Rewrite a rubric. Runs already recorded keep the instructions they used:
+     * a verdict has to keep meaning what it meant when it was given.
+     */
+    async updateJudge(judgeId, params) {
+        const res = await this._http.fetchPut(`/api/loop/judges/${encodeURIComponent(judgeId)}`, params);
+        return res.judge;
+    }
+    /**
+     * Retire a rubric. Its runs go with it; the verdicts they wrote stay. A score
+     * is evidence about an answer, and it does not stop being true because the
+     * rubric was retired.
+     */
+    async deleteJudge(judgeId) {
+        await this._http.fetchDelete(`/api/loop/judges/${encodeURIComponent(judgeId)}`);
+    }
+    /**
+     * Open a run: select the conversations now, and freeze the rubric onto them.
+     *
+     * The selection is fixed at this moment on purpose. A run whose selection is
+     * a live query silently grows as conversations arrive, so "we scored the
+     * refunds slice" becomes a claim about a set that no longer exists.
+     */
+    async startRun(judgeId, limit) {
+        const res = await this._http.fetchPost(`/api/loop/judges/${encodeURIComponent(judgeId)}/runs`, limit ? { limit } : {});
+        return res.run;
+    }
+    async listRuns(judgeId) {
+        const path = judgeId
+            ? `/api/loop/judges/${encodeURIComponent(judgeId)}/runs`
+            : '/api/loop/runs';
+        const res = await this._http.fetchGet(path);
+        return res.runs;
+    }
+    async getRun(runId) {
+        const res = await this._http.fetchGet(`/api/loop/runs/${encodeURIComponent(runId)}`);
+        return res.run;
+    }
+    /**
+     * Take the next conversations to score, each rendered into a ready-to-send
+     * prompt. Nothing is marked taken, so a caller that dies half way loses
+     * nothing: ask again and the same items come back.
+     */
+    async takeWork(runId, limit = 20) {
+        return this._http.fetchGet(`/api/loop/runs/${encodeURIComponent(runId)}/work?limit=${limit}`);
+    }
+    /**
+     * Hand the scores back. A verdict for a conversation outside this run, or
+     * scoring something the rubric never asked for, is refused and reported in
+     * `rejected` rather than silently dropped.
+     *
+     * Pass `finish` to close a run you have decided to stop early. Without it an
+     * unfinished run stays open, which is the honest state for work that was
+     * abandoned rather than completed.
+     */
+    async postVerdicts(runId, verdicts, finish = false) {
+        return this._http.fetchPost(`/api/loop/runs/${encodeURIComponent(runId)}/verdicts`, { verdicts, finish });
+    }
 }

package/dist/types.d.ts

@@ -2142,6 +2142,108 @@ export interface LoopConfigParams {
     /** Deterministic per conversation, so turns are never split. 0 < rate <= 1. */
     sample_rate?: number;
 }
+/** One thing a judge scores, separately from the others. */
+export interface LoopJudgeDimension {
+    key: string;
+    description?: string;
+}
+/** Which slice of the corpus a judge is responsible for. */
+export interface LoopJudgeSelection {
+    label?: string;
+    deployment_id?: string;
+    from?: string;
+    to?: string;
+    sample?: number;
+    /** Skip conversations that already carry a judge verdict. */
+    only_unscored?: boolean;
+}
+export interface LoopJudge {
+    id: string;
+    workspace_id: string;
+    name: string;
+    instructions: string;
+    dimensions: LoopJudgeDimension[];
+    selection: LoopJudgeSelection;
+    model: string | null;
+    write_gold: boolean;
+    enabled: boolean;
+    created_by: string | null;
+    created_at: string;
+    updated_at: string;
+}
+export interface LoopJudgeParams {
+    name: string;
+    instructions: string;
+    dimensions: LoopJudgeDimension[];
+    selection?: LoopJudgeSelection;
+    model?: string;
+    /**
+     * Let this judge write the answer that should have been given, not just a
+     * score. Off by default: a reference answer written by a model and then
+     * trained on is distillation, which is a decision you make deliberately.
+     */
+    write_gold?: boolean;
+    enabled?: boolean;
+}
+/** One pass of one rubric over one slice. */
+export interface LoopJudgeRun {
+    id: string;
+    judge_id: string;
+    judge_name?: string;
+    status: 'open' | 'done';
+    instructions: string;
+    dimensions: LoopJudgeDimension[];
+    model: string | null;
+    selected: number;
+    scored: number;
+    failed: number;
+    created_by: string | null;
+    created_at: string;
+    finished_at: string | null;
+}
+/**
+ * One conversation to score, already rendered into the prompt to send.
+ *
+ * Send `prompt` as-is. Assembling it yourself is how two callers end up giving
+ * the same rubric different instructions, and two judges given different
+ * instructions are not one judge.
+ */
+export interface LoopJudgeWorkItem {
+    trace_id: string;
+    messages: Array<{
+        role: string;
+        content?: string;
+    }>;
+    answer: string;
+    prompt: string;
+}
+export interface LoopJudgeWork {
+    run: LoopJudgeRun;
+    items: LoopJudgeWorkItem[];
+    remaining: number;
+}
+/** One scored conversation going back. */
+export interface LoopJudgeVerdict {
+    trace_id: string;
+    /** One score per dimension the rubric asked for, each between 0 and 1. */
+    scores?: Record<string, number>;
+    /** Your own combination of them. Left out, the plain mean is used. */
+    overall?: number;
+    reason?: string;
+    /** Only stored if the judge was created with `write_gold`. */
+    gold?: string;
+    /** Mark an item you could not score, instead of dropping it silently. */
+    error?: string;
+}
+export interface LoopJudgeVerdictResult {
+    run: LoopJudgeRun;
+    recorded: number;
+    failed: number;
+    rejected: Array<{
+        trace_id: string;
+        error: string;
+    }>;
+}
 export interface LoopStats {
     traces: number;
     signalled_traces: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "runbios-sdk",
-  "version": "0.2.1-dev.109",
+  "version": "0.2.1-dev.110",
   "description": "Official TypeScript SDK for the Run BiOS training and deployment platform API",
   "type": "module",
   "main": "./dist/index.js",