runbios-sdk 0.2.1-dev.106 → 0.2.1-dev.108

package/dist/index.d.ts

@@ -35,7 +35,7 @@ import { Loop } from './resources/loop.js';
  * SDK version. Sent as part of the User-Agent header.
  * Must match package.json "version" -- enforced by a contract test.
  */
-export declare const VERSION = "0.2.1-dev.106";
+export declare const VERSION = "0.2.1-dev.108";
 export declare class RunBiOS {
     /** Search models, fetch configs, check adapter compatibility. */
     readonly models: Models;

package/dist/index.js

@@ -35,7 +35,7 @@ import { Loop } from './resources/loop.js';
  * SDK version. Sent as part of the User-Agent header.
  * Must match package.json "version" -- enforced by a contract test.
  */
-export const VERSION = '0.2.1-dev.106';
+export const VERSION = '0.2.1-dev.108';
 export class RunBiOS {
     /** Search models, fetch configs, check adapter compatibility. */
     models;

package/dist/resources/loop.d.ts

@@ -1,5 +1,5 @@
 import type { HttpClient } from '../client.js';
-import type { LoopTrace, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopSignal, LoopSignalParams, LoopCandidate, LoopCandidateParams, LoopDataset, LoopDatasetListParams, LoopDatasetItem, LoopDatasetCreateParams, LoopConfig, LoopConfigParams, LoopStats } from '../types.js';
+import type { LoopTrace, LoopTraceListResponse, LoopTraceListParams, LoopCaptureParams, LoopCaptureResult, LoopSignal, LoopSignalParams, LoopCandidate, LoopCandidateParams, LoopGrader, LoopGraderParams, LoopGradeResult, LoopDataset, LoopDatasetListParams, LoopDatasetItem, LoopDatasetCreateParams, LoopConfig, LoopConfigParams, LoopStats } from '../types.js';
 /**
  * The Conscious Loop -- capture what your model was asked and answered, record
  * whether it was right, and turn those judgements into training data.
@@ -88,6 +88,42 @@ export declare class Loop {
      * behavioural hint.
      */
     signal(traceId: string, params: LoopSignalParams): Promise<LoopSignal>;
+    /**
+     * Thumbs up: this answer was good. Shorthand for `signal(id, { verdict:
+     * 'accepted' })` — there is one stored vocabulary behind both, so a thumbs-up
+     * and an "accepted" can never disagree in your corpus.
+     */
+    upvote(traceId: string, reason?: string): Promise<LoopSignal>;
+    /**
+     * Thumbs down: this answer was bad, and you have nothing better to offer.
+     *
+     * On its own this REMOVES the answer from training rather than teaching
+     * anything — there is no better version to learn. If you know what it should
+     * have said, `correct()` is worth far more.
+     */
+    downvote(traceId: string, reason?: string): Promise<LoopSignal>;
+    /**
+     * The model was wrong; here is what it should have said.
+     *
+     * The most valuable feedback there is, because it produces BOTH halves of a
+     * preference pair from one action: the model learns your answer and learns to
+     * avoid its own.
+     */
+    correct(traceId: string, answer: string, reason?: string): Promise<LoopSignal>;
+    /**
+     * Record the GOLD answer for this question — the reference, regardless of
+     * what the model happened to say.
+     *
+     * Different from `correct()` in a way that matters: a correction asserts the
+     * model was wrong, so the original becomes the rejected half of a pair. A gold
+     * answer asserts nothing about the model, so if it MATCHES what was said, no
+     * preference pair is invented — but SFT still learns it.
+     *
+     * It is the most reusable thing you can record: it trains SFT, forms a DPO
+     * pair when it differs from the answer, and stands in as the GRPO reference
+     * when you have not supplied a separate ground truth.
+     */
+    gold(traceId: string, answer: string, reason?: string): Promise<LoopSignal>;
     /** Every verdict on one conversation, oldest first. */
     listSignals(traceId: string): Promise<LoopSignal[]>;
     /**
@@ -172,6 +208,47 @@ export declare class Loop {
         deleted: boolean;
         dataset_id: string;
     }>;
+    /**
+     * Write a rule that scores answers without a person.
+     *
+     * Human review is the most trustworthy feedback and the least available. A
+     * grader is written once and applied to every answer afterwards: did it
+     * contain the required phrase, did it parse as the schema you asked for, is
+     * the number within tolerance of the known answer, did it call the function
+     * it should have.
+     *
+     * Every kind here is DETERMINISTIC -- no model call, no network. That is why
+     * a verifier outranks a judge when they disagree: it cannot be flattered and
+     * it cannot drift between runs.
+     *
+     * `weight` is the multiplier: a criterion that matters twice as much gets
+     * twice the weight, and the combined score is the weighted mean over the
+     * rules that actually applied.
+     *
+     * A caution worth knowing before you write a set: a rule made only of
+     * `forbidden` phrases is satisfied VACUOUSLY by an answer that says nothing.
+     * Pair it with a `required` phrase, or you are rewarding silence.
+     */
+    createGrader(params: LoopGraderParams): Promise<LoopGrader>;
+    /** Every rule this workspace has written. */
+    listGraders(): Promise<LoopGrader[]>;
+    deleteGrader(id: string): Promise<{
+        deleted: boolean;
+        grader_id: string;
+    }>;
+    /**
+     * Apply this workspace's rules to one captured answer AND to every
+     * alternative sampled for it.
+     *
+     * Grading both in one pass is the point: scoring only the original gives a
+     * verdict, while scoring the samples as well gives the preference pair. Sample
+     * your model, call this, and you have DPO data with nobody reading anything.
+     *
+     * A run where no rule could apply writes NOTHING -- no verdict, no scores.
+     * Recording a zero that no rule produced would poison curation with a
+     * judgement nobody reached, so `applied: 0` is reported instead.
+     */
+    grade(traceId: string): Promise<LoopGradeResult>;
     /** Every source this workspace has configured. */
     listConfigs(): Promise<LoopConfig[]>;
     /**

package/dist/resources/loop.js

@@ -116,6 +116,50 @@ export class Loop {
         const res = await this._http.fetchPost(`/api/loop/traces/${encodeURIComponent(traceId)}/signals`, params);
         return res.signal;
     }
+    /**
+     * Thumbs up: this answer was good. Shorthand for `signal(id, { verdict:
+     * 'accepted' })` — there is one stored vocabulary behind both, so a thumbs-up
+     * and an "accepted" can never disagree in your corpus.
+     */
+    async upvote(traceId, reason) {
+        return this.signal(traceId, { verdict: 'accepted', reason });
+    }
+    /**
+     * Thumbs down: this answer was bad, and you have nothing better to offer.
+     *
+     * On its own this REMOVES the answer from training rather than teaching
+     * anything — there is no better version to learn. If you know what it should
+     * have said, `correct()` is worth far more.
+     */
+    async downvote(traceId, reason) {
+        return this.signal(traceId, { verdict: 'rejected', reason });
+    }
+    /**
+     * The model was wrong; here is what it should have said.
+     *
+     * The most valuable feedback there is, because it produces BOTH halves of a
+     * preference pair from one action: the model learns your answer and learns to
+     * avoid its own.
+     */
+    async correct(traceId, answer, reason) {
+        return this.signal(traceId, { verdict: 'edited', correction: answer, reason });
+    }
+    /**
+     * Record the GOLD answer for this question — the reference, regardless of
+     * what the model happened to say.
+     *
+     * Different from `correct()` in a way that matters: a correction asserts the
+     * model was wrong, so the original becomes the rejected half of a pair. A gold
+     * answer asserts nothing about the model, so if it MATCHES what was said, no
+     * preference pair is invented — but SFT still learns it.
+     *
+     * It is the most reusable thing you can record: it trains SFT, forms a DPO
+     * pair when it differs from the answer, and stands in as the GRPO reference
+     * when you have not supplied a separate ground truth.
+     */
+    async gold(traceId, answer, reason) {
+        return this.signal(traceId, { verdict: 'gold', correction: answer, reason });
+    }
     /** Every verdict on one conversation, oldest first. */
     async listSignals(traceId) {
         const res = await this._http.fetchGet(`/api/loop/traces/${encodeURIComponent(traceId)}/signals`);
@@ -236,6 +280,55 @@ export class Loop {
     async deleteDataset(id) {
         return this._http.fetchDelete(`/api/loop/datasets/${encodeURIComponent(id)}`);
     }
+    // ── graders ───────────────────────────────────────────────────────────
+    /**
+     * Write a rule that scores answers without a person.
+     *
+     * Human review is the most trustworthy feedback and the least available. A
+     * grader is written once and applied to every answer afterwards: did it
+     * contain the required phrase, did it parse as the schema you asked for, is
+     * the number within tolerance of the known answer, did it call the function
+     * it should have.
+     *
+     * Every kind here is DETERMINISTIC -- no model call, no network. That is why
+     * a verifier outranks a judge when they disagree: it cannot be flattered and
+     * it cannot drift between runs.
+     *
+     * `weight` is the multiplier: a criterion that matters twice as much gets
+     * twice the weight, and the combined score is the weighted mean over the
+     * rules that actually applied.
+     *
+     * A caution worth knowing before you write a set: a rule made only of
+     * `forbidden` phrases is satisfied VACUOUSLY by an answer that says nothing.
+     * Pair it with a `required` phrase, or you are rewarding silence.
+     */
+    async createGrader(params) {
+        const res = await this._http.fetchPost('/api/loop/graders', params);
+        return res.grader;
+    }
+    /** Every rule this workspace has written. */
+    async listGraders() {
+        const res = await this._http.fetchGet('/api/loop/graders');
+        return res.graders;
+    }
+    async deleteGrader(id) {
+        return this._http.fetchDelete(`/api/loop/graders/${encodeURIComponent(id)}`);
+    }
+    /**
+     * Apply this workspace's rules to one captured answer AND to every
+     * alternative sampled for it.
+     *
+     * Grading both in one pass is the point: scoring only the original gives a
+     * verdict, while scoring the samples as well gives the preference pair. Sample
+     * your model, call this, and you have DPO data with nobody reading anything.
+     *
+     * A run where no rule could apply writes NOTHING -- no verdict, no scores.
+     * Recording a zero that no rule produced would poison curation with a
+     * judgement nobody reached, so `applied: 0` is reported instead.
+     */
+    async grade(traceId) {
+        return this._http.fetchPost(`/api/loop/traces/${encodeURIComponent(traceId)}/grade`);
+    }
     // ── capture settings ──────────────────────────────────────────────────
     /** Every source this workspace has configured. */
     async listConfigs() {

package/dist/types.d.ts

@@ -1960,7 +1960,15 @@ export type LoopMethod = 'sft' | 'dpo' | 'grpo';
 /** Who judged an answer. */
 export type LoopSource = 'human' | 'verifier' | 'judge' | 'behavioural';
 /** The judgement itself. */
-export type LoopVerdict = 'accepted' | 'rejected' | 'edited' | 'scored';
+/**
+ * The judgement recorded on an answer.
+ *
+ * `accepted` / `rejected` are the thumbs (see `upvote` / `downvote`).
+ * `edited` says the model was WRONG and carries the better answer.
+ * `gold` records the reference answer for the question, making no claim about
+ * whether the model was right — which is why it is separate from `edited`.
+ */
+export type LoopVerdict = 'accepted' | 'rejected' | 'edited' | 'scored' | 'gold';
 /** A function an assistant turn asked to invoke. `arguments` is JSON-encoded. */
 export interface LoopToolCall {
     id?: string;
@@ -2167,3 +2175,65 @@ export interface LoopCandidate {
     metadata: Record<string, unknown>;
     created_at: string;
 }
+/** The deterministic checks a grader can perform. */
+export type LoopGraderKind = 'exact_match' | 'contains' | 'regex' | 'json_valid' | 'numeric' | 'tool_called';
+export interface LoopGraderConfig {
+    expected?: string;
+    /** Every phrase that must appear. */
+    required?: string[];
+    /** Phrases that must not. On their own these are passed by silence. */
+    forbidden?: string[];
+    pattern?: string;
+    /** Keys the answer must carry, for `json_valid`. */
+    keys?: string[];
+    /** How far from `expected` still counts, for `numeric`. */
+    tolerance?: number;
+    /** The function that must have been called, for `tool_called`. */
+    function?: string;
+    case_sensitive?: boolean;
+}
+export interface LoopGraderParams {
+    name: string;
+    kind: LoopGraderKind;
+    config: LoopGraderConfig;
+    /** The multiplier. Twice as important, twice the weight. Must exceed zero. */
+    weight?: number;
+    enabled?: boolean;
+    /** Limit the rule to one capture source. Omit to apply it everywhere. */
+    deployment_id?: string;
+}
+export interface LoopGrader extends LoopGraderParams {
+    id: string;
+    workspace_id: string;
+    weight: number;
+    enabled: boolean;
+    created_by: string | null;
+    created_at: string;
+    updated_at: string;
+}
+export interface LoopGraderResult {
+    grader_id: string;
+    name: string;
+    kind: string;
+    weight: number;
+    passed: boolean;
+    score: number;
+    /** What happened, in words -- "failed" alone sends you to read the rule. */
+    detail: string;
+    /** The rule had no opinion. Excluded from the score rather than counted 0. */
+    skipped: boolean;
+}
+export interface LoopGradeReport {
+    /** Weighted mean over the rules that applied, 0 to 1. */
+    score: number;
+    results: LoopGraderResult[];
+    /** The denominator. 1.0 from one rule is not 1.0 from six. */
+    applied: number;
+    skipped: number;
+}
+export interface LoopGradeResult {
+    report: LoopGradeReport;
+    /** Null when no rule applied, because no verdict was invented. */
+    signal_id: string | null;
+    candidates_graded: number;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "runbios-sdk",
-  "version": "0.2.1-dev.106",
+  "version": "0.2.1-dev.108",
   "description": "Official TypeScript SDK for the Run BiOS training and deployment platform API",
   "type": "module",
   "main": "./dist/index.js",