synthiaresearch 0.0.1

package/README.md

@@ -0,0 +1,26 @@
+# synthiaresearch
+
+JavaScript/TypeScript SDK for the Synthia API.
+
+```bash
+npm install synthiaresearch
+```
+
+```ts
+import { Synthia } from "synthiaresearch";
+
+const synthia = new Synthia(); // reads SYNTHIA_API_KEY / SYNTHIA_BASE_URL
+
+const { dataset } = await synthia.prepare(async (probe) => {
+  return myAgent.respond(probe);
+});
+
+const results = await dataset.rollout(async (transcript, sandbox) => {
+  return myAgent.respondWithTools(transcript, sandbox);
+});
+
+const check = await synthia.rollouts.qualityCheck(results);
+await check.wait({ verbose: true });
+```
+
+Requires Node 18+. Mirrors the Python SDK (`synthiaresearch` on PyPI).

package/dist/client.d.ts

@@ -0,0 +1,298 @@
+export declare const DEFAULT_BASE_URL = "https://synthia-research--synthia-api-web.modal.run";
+/** One traced tool invocation made by the agent while answering a probe. */
+export interface ToolCall {
+    name: string;
+    input: Record<string, unknown>;
+    output?: Record<string, unknown> | null;
+    is_error?: boolean;
+}
+/** An agent's reply plus the tool calls it made producing it. */
+export interface AgentReply {
+    reply: string;
+    tool_calls?: ToolCall[];
+}
+export type Agent = (probe: string) => string | AgentReply | Promise<string | AgentReply>;
+export interface SandboxConfig {
+    seed: number;
+    fail_tools: string[];
+    state: Record<string, unknown>;
+}
+export interface ToolEvent {
+    name: string;
+    input: Record<string, unknown>;
+    output: Record<string, unknown>;
+    is_error: boolean;
+}
+/**
+ * Local replica of the server's deterministic tool sandbox.
+ *
+ * Must stay hash-identical to synthia_api.pipeline.rollout.ToolSandbox:
+ * outputs are a pure function of (tool name, input, state version, seed),
+ * which is how the server reproduces the same tool behavior from the
+ * events the SDK reports. Tool inputs must be JSON-native values that
+ * round-trip exactly.
+ */
+export declare class ToolSandbox {
+    seed: number;
+    failTools: Set<string>;
+    state: Record<string, unknown>;
+    events: ToolEvent[];
+    constructor(seed: number, failTools?: Set<string>, state?: Record<string, unknown>);
+    static fromConfig(config: SandboxConfig): ToolSandbox;
+    call(name: string, toolInput: Record<string, unknown>): Record<string, unknown>;
+}
+export interface TranscriptTurn {
+    role: string;
+    content: string;
+}
+export type RolloutAgent = (transcript: TranscriptTurn[], sandbox: ToolSandbox) => string | Promise<string>;
+declare class Http {
+    private baseUrl;
+    headers: Record<string, string>;
+    /** Gate awaited before every request; Synthia points it at the session
+     * handshake so all later requests carry the session headers. */
+    ready: Promise<void>;
+    constructor(baseUrl: string, headers: Record<string, string>);
+    get(path: string, params?: Record<string, string | number>): Promise<any>;
+    post(path: string, body?: unknown): Promise<any>;
+    /** Un-gated request returning the raw Response (used by the handshake). */
+    raw(method: string, path: string, body?: unknown, params?: Record<string, string | number>): Promise<Response>;
+}
+export interface UserModel {
+    id: string;
+    probe_session_id: string;
+    persona: string;
+    traits: string[];
+    representation_id: string | null;
+}
+export interface WaitOptions {
+    pollInterval?: number;
+    timeout?: number;
+    verbose?: boolean;
+}
+/**
+ * A validation of a dataset: a per-scenario judge gate plus collective
+ * fidelity/diversity reports and an advisory verdict.
+ */
+export declare class ValidationRun {
+    #private;
+    id: string;
+    dataset_id: string;
+    status: string;
+    verdict: string | null;
+    reference: Record<string, unknown> | null;
+    validity: Record<string, unknown> | null;
+    fidelity: Record<string, unknown> | null;
+    diversity: Record<string, unknown> | null;
+    error: string | null;
+    constructor(data: any, http: Http);
+    /** Poll until validation finishes; verbose prints server telemetry. */
+    wait(opts?: WaitOptions): Promise<ValidationRun>;
+    /** Per-scenario judge verdicts: scenario_id, passed, judge. */
+    scenarios(): Promise<any[]>;
+}
+export declare class Dataset {
+    #private;
+    id: string;
+    generation_id: string;
+    user_model_id: string;
+    row_count: number;
+    constructor(data: any, http: Http);
+    download(): Promise<any[]>;
+    /** Start an async validation run over this dataset. */
+    validate(): Promise<ValidationRun>;
+    /** Play this dataset's scenarios against a local agent. */
+    rollout(agent: RolloutAgent, opts?: RunOptions): Promise<RolloutResult[]>;
+}
+export declare class GenerationJob {
+    #private;
+    id: string;
+    status: string;
+    user_model_id: string;
+    count: number;
+    dataset_id: string | null;
+    error: string | null;
+    constructor(data: any, http: Http);
+    /** Poll until the job finishes; verbose prints server telemetry live. */
+    wait(opts?: WaitOptions): Promise<Dataset>;
+}
+export declare class Seeds {
+    #private;
+    constructor(http: Http);
+    /** Upload seed material (documents, tool schemas, policies, traces...). */
+    ingest(opts: {
+        kind: string;
+        source: string;
+        content: Record<string, unknown>;
+        version?: string;
+        metadata?: Record<string, unknown>;
+    }): Promise<any>;
+}
+export declare class UserModels {
+    #private;
+    constructor(http: Http);
+    /**
+     * Probe `agent` until the server converges on a user model.
+     *
+     * The agent runs locally; only probe questions, replies, and traced
+     * tool calls travel over the wire. `verbose` prints the server's
+     * telemetry (probe decisions, ingestion, inference) after each turn.
+     */
+    createFromProbe(agent: Agent, opts?: {
+        maxTurns?: number;
+        verbose?: boolean;
+    }): Promise<UserModel>;
+    get(modelId: string): Promise<UserModel>;
+    list(session?: string): Promise<UserModel[]>;
+}
+export declare class Datasets {
+    #private;
+    constructor(http: Http);
+    get(datasetId: string): Promise<Dataset>;
+    /** Datasets newest first; `session` filters to one SDK session. */
+    list(session?: string): Promise<Dataset[]>;
+    /**
+     * Start a generation job. qualityCheckId names a completed quality
+     * check whose results calibrate the batch's difficulty and coverage.
+     */
+    generate(userModel: UserModel | string, opts?: {
+        count?: number;
+        qualityCheckId?: string;
+    }): Promise<GenerationJob>;
+}
+/**
+ * Outcome of Synthia.prepare(): the dataset to roll out, the user model
+ * behind it, and how the decision was made.
+ */
+export interface PrepareResult {
+    dataset: Dataset;
+    userModel: UserModel;
+    action: "generated" | "reused";
+    reason: string;
+    successRate: number | null;
+    qualityCheckId: string | null;
+}
+/**
+ * An async evaluation of finished rollouts: per rollout, the server
+ * analyzes the agent's state trajectory and judges pass/fail. The
+ * per-rollout results are the product; there is no aggregate verdict.
+ */
+export declare class QualityCheck {
+    #private;
+    id: string;
+    status: string;
+    rollout_ids: string[];
+    error: string | null;
+    constructor(data: any, http: Http);
+    /** Poll until the check finishes; verbose prints server telemetry. */
+    wait(opts?: WaitOptions): Promise<QualityCheck>;
+    /**
+     * Per-rollout results: rollout_id, passed, states (the agentic-state
+     * trajectory), and judge (dimensions + issues).
+     */
+    rollouts(): Promise<any[]>;
+}
+/**
+ * One finished rollout: the conversation a scenario produced, plus every
+ * tool call the agent made along the way (each tagged with its turn_idx).
+ */
+export interface RolloutResult {
+    rollout_id: string;
+    scenario_id: string;
+    status: string;
+    turns: number;
+    transcript: TranscriptTurn[];
+    tool_events: any[];
+}
+export interface RunOptions {
+    maxTurns?: number;
+    concurrency?: number;
+}
+export declare class Rollouts {
+    #private;
+    constructor(http: Http, sessionId?: string | null);
+    /**
+     * A stored rollout's full captured state: status, seed, transcript,
+     * tool events, and sandbox.
+     */
+    get(rolloutId: string): Promise<any>;
+    /**
+     * Play a dataset's scenarios against `agent` (most recent dataset when
+     * none is given).
+     *
+     * The agent runs locally: each turn it gets the transcript so far and
+     * a deterministic ToolSandbox for its tool calls; only its reply and
+     * tool events travel over the wire. Scenarios run with `concurrency`
+     * in-flight at once (turns within one conversation are sequential) —
+     * pass concurrency: 1 to run them strictly one at a time.
+     */
+    run(agent: RolloutAgent, dataset?: Dataset | string | null, opts?: RunOptions): Promise<RolloutResult[]>;
+    /**
+     * Start an async quality check over finished rollouts: the server
+     * analyzes each rollout's agentic states in parallel and judges
+     * whether the agent passed each scenario.
+     */
+    qualityCheck(rollouts: (RolloutResult | string)[]): Promise<QualityCheck>;
+    /** Run one rollout session; one HTTP round-trip per agent turn. */
+    runScenario(agent: RolloutAgent, scenarioId: string, opts?: {
+        maxTurns?: number;
+        randomSeed?: number;
+    }): Promise<RolloutResult>;
+}
+export interface SynthiaOptions {
+    apiKey?: string;
+    baseUrl?: string;
+    session?: string | false;
+}
+export interface PrepareOptions {
+    count?: number;
+    maxTurns?: number;
+    minSuccessRate?: number;
+    maxSuccessRate?: number;
+    verbose?: boolean;
+}
+/**
+ * Client entry point.
+ *
+ * Session identity: every client belongs to a named session — the stable,
+ * account-scoped identity of one script, persisted across executions
+ * (same name resumes the same session; re-runs reuse its datasets instead
+ * of re-probing/re-generating). Resolution order: `session` option >
+ * SYNTHIA_SESSION env var > derived "project/script" name from the entry
+ * point. `session: false` opts out into a fresh ephemeral session.
+ *
+ * Degradation: an old server without /v1/sdk-sessions -> no session;
+ * keyless against a keyed server -> anonymous session; an invalid apiKey
+ * fails on first use with the server's message.
+ */
+export declare class Synthia {
+    #private;
+    sessionName: string;
+    sessionId: string | null;
+    invocationId: string | null;
+    seeds: Seeds;
+    userModels: UserModels;
+    datasets: Datasets;
+    rollouts: Rollouts;
+    constructor(options?: SynthiaOptions);
+    /**
+     * Probe + generate only when needed; otherwise reuse the latest dataset.
+     *
+     * The main entry point for the probe and generation steps. `count` is
+     * exact: the returned dataset has exactly that many rows, so reuse
+     * requires the latest dataset to match it in addition to the quality
+     * gate. Probing and generation run only when no dataset exists yet,
+     * when the row count differs (generation-only — the session's probed
+     * user model is reused), or when the latest completed quality check's
+     * pass rate falls outside [minSuccessRate, maxSuccessRate].
+     * Out-of-band regeneration passes that quality check to the server,
+     * which feeds its real results into scenario generation so the new
+     * batch recalibrates difficulty and coverage.
+     *
+     * All lookups are scoped to this client's session: re-running the same
+     * script reuses its own dataset, and drift signals from other
+     * scripts/sessions never trigger regeneration here.
+     */
+    prepare(agent: Agent, opts?: PrepareOptions): Promise<PrepareResult>;
+}
+export {};

package/dist/client.js

@@ -0,0 +1,729 @@
+import { createHash, randomUUID } from "node:crypto";
+import { basename, dirname, extname, resolve } from "node:path";
+export const DEFAULT_BASE_URL = "https://synthia-research--synthia-api-web.modal.run";
+const SDK_VERSION = "0.0.1";
+// Launcher argv[1] stems that don't identify a script (REPLs, runners).
+const INTERACTIVE_STEMS = new Set(["", "node", "npx", "tsx", "ts-node"]);
+/**
+ * Stable session name derived from the entry-point script.
+ *
+ * Last two path components only ("project/script"): readable, survives
+ * moving the project, and never leaks the full directory layout. Falls
+ * back to the working directory's name for REPLs.
+ */
+function defaultSessionName() {
+    const argv1 = process.argv[1] ?? "";
+    const stem = argv1 ? basename(argv1, extname(argv1)) : "";
+    if (!argv1 || INTERACTIVE_STEMS.has(stem))
+        return basename(process.cwd());
+    const p = resolve(argv1);
+    return `${basename(dirname(p))}/${stem}`;
+}
+/**
+ * JSON serialization matching Python's json.dumps(v, sort_keys=True):
+ * sorted keys, ", "/": " separators, non-ASCII escaped. Required so
+ * ToolSandbox hashes agree with the server. Caveat: JS has one number
+ * type, so Python's 1 vs 1.0 distinction cannot be reproduced here.
+ */
+function pyJson(v) {
+    if (v === null || v === undefined)
+        return "null";
+    if (typeof v === "boolean")
+        return v ? "true" : "false";
+    if (typeof v === "number")
+        return String(v);
+    if (typeof v === "string")
+        return pyString(v);
+    if (Array.isArray(v))
+        return "[" + v.map(pyJson).join(", ") + "]";
+    const obj = v;
+    const keys = Object.keys(obj).sort();
+    return ("{" + keys.map((k) => `${pyString(k)}: ${pyJson(obj[k])}`).join(", ") + "}");
+}
+function pyString(s) {
+    return JSON.stringify(s).replace(/[\u0080-\uffff]/g, (c) => "\\u" + c.charCodeAt(0).toString(16).padStart(4, "0"));
+}
+/**
+ * Local replica of the server's deterministic tool sandbox.
+ *
+ * Must stay hash-identical to synthia_api.pipeline.rollout.ToolSandbox:
+ * outputs are a pure function of (tool name, input, state version, seed),
+ * which is how the server reproduces the same tool behavior from the
+ * events the SDK reports. Tool inputs must be JSON-native values that
+ * round-trip exactly.
+ */
+export class ToolSandbox {
+    seed;
+    failTools;
+    state;
+    events = [];
+    constructor(seed, failTools, state) {
+        this.seed = seed;
+        this.failTools = failTools ?? new Set();
+        this.state = { ...(state ?? {}) };
+    }
+    static fromConfig(config) {
+        return new ToolSandbox(config.seed, new Set(config.fail_tools), config.state);
+    }
+    call(name, toolInput) {
+        const isError = this.failTools.has(name);
+        let output;
+        if (isError) {
+            output = { error: `${name} is unavailable` };
+        }
+        else {
+            const version = this.state["version"] ?? 0;
+            const digest = createHash("sha256")
+                .update(pyJson([name, toolInput, version, this.seed]))
+                .digest("hex")
+                .slice(0, 8);
+            output = { ok: true, result_id: digest };
+            this.state["version"] = version + 1;
+            this.state[`last_${name}`] = digest;
+        }
+        this.events.push({ name, input: toolInput, output, is_error: isError });
+        return output;
+    }
+}
+class HttpError extends Error {
+    status;
+    body;
+    constructor(status, body, method, path) {
+        super(`${method} ${path} failed with ${status}: ${body}`);
+        this.status = status;
+        this.body = body;
+    }
+}
+class Http {
+    baseUrl;
+    headers;
+    /** Gate awaited before every request; Synthia points it at the session
+     * handshake so all later requests carry the session headers. */
+    ready = Promise.resolve();
+    constructor(baseUrl, headers) {
+        this.baseUrl = baseUrl;
+        this.headers = headers;
+    }
+    async get(path, params) {
+        await this.ready;
+        const r = await this.raw("GET", path, undefined, params);
+        if (!r.ok)
+            throw new HttpError(r.status, await r.text(), "GET", path);
+        return r.json();
+    }
+    async post(path, body) {
+        await this.ready;
+        const r = await this.raw("POST", path, body);
+        if (!r.ok)
+            throw new HttpError(r.status, await r.text(), "POST", path);
+        return r.json();
+    }
+    /** Un-gated request returning the raw Response (used by the handshake). */
+    async raw(method, path, body, params) {
+        const url = new URL(this.baseUrl + path);
+        for (const [k, v] of Object.entries(params ?? {})) {
+            url.searchParams.set(k, String(v));
+        }
+        return fetch(url, {
+            method,
+            headers: { "content-type": "application/json", ...this.headers },
+            body: body === undefined ? undefined : JSON.stringify(body),
+            // Generous timeout: probe convergence runs model inference server-side.
+            signal: AbortSignal.timeout(300_000),
+        });
+    }
+}
+const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
+/** Cursor-based poller that prints a run's server-side telemetry. */
+class EventStream {
+    http;
+    path;
+    enabled;
+    #after = 0;
+    constructor(http, path, enabled) {
+        this.http = http;
+        this.path = path;
+        this.enabled = enabled;
+    }
+    async pump() {
+        if (!this.enabled)
+            return;
+        const body = await this.http.get(this.path, { after: this.#after });
+        for (const event of body.data) {
+            this.#after = event.seq;
+            const detail = Object.entries(event.data)
+                .map(([k, v]) => `${k}=${v}`)
+                .join(" ");
+            console.log(`    ~ [${event.stage}] ${event.message}` +
+                (detail ? `  (${detail})` : ""));
+        }
+    }
+}
+/**
+ * A validation of a dataset: a per-scenario judge gate plus collective
+ * fidelity/diversity reports and an advisory verdict.
+ */
+export class ValidationRun {
+    id;
+    dataset_id;
+    status;
+    verdict;
+    reference;
+    validity;
+    fidelity;
+    diversity;
+    error;
+    #http;
+    constructor(data, http) {
+        this.id = data.id;
+        this.dataset_id = data.dataset_id;
+        this.status = data.status;
+        this.verdict = data.verdict;
+        this.reference = data.reference;
+        this.validity = data.validity;
+        this.fidelity = data.fidelity;
+        this.diversity = data.diversity;
+        this.error = data.error;
+        this.#http = http;
+    }
+    /** Poll until validation finishes; verbose prints server telemetry. */
+    async wait(opts = {}) {
+        const { pollInterval = 2, timeout = 1800, verbose = false } = opts;
+        const events = new EventStream(this.#http, `/v1/validation-runs/${this.id}/events`, verbose);
+        const deadline = Date.now() + timeout * 1000;
+        while (this.status === "running") {
+            if (Date.now() > deadline) {
+                throw new Error(`validation run ${this.id} still running after ${timeout}s`);
+            }
+            await sleep(pollInterval * 1000);
+            const data = await this.#http.get(`/v1/validation-runs/${this.id}`);
+            this.status = data.status;
+            this.verdict = data.verdict;
+            this.reference = data.reference;
+            this.validity = data.validity;
+            this.fidelity = data.fidelity;
+            this.diversity = data.diversity;
+            this.error = data.error;
+            await events.pump();
+        }
+        await events.pump(); // catch events written after the final status poll
+        if (this.status !== "succeeded") {
+            throw new Error(`validation run ${this.id} failed: ${this.error}`);
+        }
+        return this;
+    }
+    /** Per-scenario judge verdicts: scenario_id, passed, judge. */
+    async scenarios() {
+        const body = await this.#http.get(`/v1/validation-runs/${this.id}/scenarios`);
+        return body.data;
+    }
+}
+export class Dataset {
+    id;
+    generation_id;
+    user_model_id;
+    row_count;
+    #http;
+    constructor(data, http) {
+        this.id = data.id;
+        this.generation_id = data.generation_id;
+        this.user_model_id = data.user_model_id;
+        this.row_count = data.row_count;
+        this.#http = http;
+    }
+    async download() {
+        const body = await this.#http.get(`/v1/datasets/${this.id}/rows`);
+        return body.data;
+    }
+    /** Start an async validation run over this dataset. */
+    async validate() {
+        const data = await this.#http.post(`/v1/datasets/${this.id}/validations`);
+        return new ValidationRun(data, this.#http);
+    }
+    /** Play this dataset's scenarios against a local agent. */
+    async rollout(agent, opts = {}) {
+        return new Rollouts(this.#http).run(agent, this, opts);
+    }
+}
+export class GenerationJob {
+    id;
+    status;
+    user_model_id;
+    count;
+    dataset_id;
+    error;
+    #http;
+    constructor(data, http) {
+        this.id = data.id;
+        this.status = data.status;
+        this.user_model_id = data.user_model_id;
+        this.count = data.count;
+        this.dataset_id = data.dataset_id;
+        this.error = data.error;
+        this.#http = http;
+    }
+    /** Poll until the job finishes; verbose prints server telemetry live. */
+    async wait(opts = {}) {
+        const { pollInterval = 2, timeout = 1800, verbose = false } = opts;
+        const events = new EventStream(this.#http, `/v1/generations/${this.id}/events`, verbose);
+        const deadline = Date.now() + timeout * 1000;
+        while (this.status === "running") {
+            if (Date.now() > deadline) {
+                throw new Error(`generation ${this.id} still running after ${timeout}s`);
+            }
+            await sleep(pollInterval * 1000);
+            const data = await this.#http.get(`/v1/generations/${this.id}`);
+            this.status = data.status;
+            this.dataset_id = data.dataset_id;
+            this.error = data.error;
+            await events.pump();
+        }
+        await events.pump(); // catch events written after the final status poll
+        if (this.status !== "succeeded") {
+            throw new Error(`generation ${this.id} failed: ${this.error}`);
+        }
+        const data = await this.#http.get(`/v1/datasets/${this.dataset_id}`);
+        return new Dataset(data, this.#http);
+    }
+}
+export class Seeds {
+    #http;
+    constructor(http) {
+        this.#http = http;
+    }
+    /** Upload seed material (documents, tool schemas, policies, traces...). */
+    async ingest(opts) {
+        return this.#http.post("/v1/seeds", {
+            kind: opts.kind,
+            source: opts.source,
+            content: opts.content,
+            version: opts.version ?? "1",
+            metadata: opts.metadata ?? {},
+        });
+    }
+}
+export class UserModels {
+    #http;
+    constructor(http) {
+        this.#http = http;
+    }
+    /**
+     * Probe `agent` until the server converges on a user model.
+     *
+     * The agent runs locally; only probe questions, replies, and traced
+     * tool calls travel over the wire. `verbose` prints the server's
+     * telemetry (probe decisions, ingestion, inference) after each turn.
+     */
+    async createFromProbe(agent, opts = {}) {
+        const { maxTurns = 10, verbose = false } = opts;
+        let session = await this.#http.post("/v1/probe-sessions", {
+            max_turns: maxTurns,
+        });
+        const events = new EventStream(this.#http, `/v1/probe-sessions/${session.id}/events`, verbose);
+        while (session.status === "active") {
+            const raw = await agent(session.next_probe);
+            const reply = typeof raw === "string" ? raw : raw.reply;
+            const toolCalls = typeof raw === "string"
+                ? []
+                : (raw.tool_calls ?? []).map((tc) => ({
+                    name: tc.name,
+                    input: tc.input,
+                    output: tc.output ?? null,
+                    is_error: tc.is_error ?? false,
+                }));
+            session = await this.#http.post(`/v1/probe-sessions/${session.id}/responses`, { reply, tool_calls: toolCalls });
+            await events.pump();
+        }
+        return this.get(session.user_model_id);
+    }
+    async get(modelId) {
+        return this.#http.get(`/v1/user-models/${modelId}`);
+    }
+    async list(session) {
+        const params = session ? { sdk_session: session } : undefined;
+        const body = await this.#http.get("/v1/user-models", params);
+        return body.data;
+    }
+}
+export class Datasets {
+    #http;
+    constructor(http) {
+        this.#http = http;
+    }
+    async get(datasetId) {
+        const data = await this.#http.get(`/v1/datasets/${datasetId}`);
+        return new Dataset(data, this.#http);
+    }
+    /** Datasets newest first; `session` filters to one SDK session. */
+    async list(session) {
+        const params = session ? { sdk_session: session } : undefined;
+        const body = await this.#http.get("/v1/datasets", params);
+        return body.data.map((d) => new Dataset(d, this.#http));
+    }
+    /**
+     * Start a generation job. qualityCheckId names a completed quality
+     * check whose results calibrate the batch's difficulty and coverage.
+     */
+    async generate(userModel, opts = {}) {
+        const { count = 20, qualityCheckId } = opts;
+        const modelId = typeof userModel === "string" ? userModel : userModel.id;
+        const body = {
+            user_model_id: modelId,
+            count,
+        };
+        if (qualityCheckId)
+            body["quality_check_id"] = qualityCheckId;
+        const data = await this.#http.post("/v1/generations", body);
+        return new GenerationJob(data, this.#http);
+    }
+}
+/**
+ * An async evaluation of finished rollouts: per rollout, the server
+ * analyzes the agent's state trajectory and judges pass/fail. The
+ * per-rollout results are the product; there is no aggregate verdict.
+ */
+export class QualityCheck {
+    id;
+    status;
+    rollout_ids;
+    error;
+    #http;
+    constructor(data, http) {
+        this.id = data.id;
+        this.status = data.status;
+        this.rollout_ids = data.rollout_ids;
+        this.error = data.error;
+        this.#http = http;
+    }
+    /** Poll until the check finishes; verbose prints server telemetry. */
+    async wait(opts = {}) {
+        const { pollInterval = 2, timeout = 1800, verbose = false } = opts;
+        const events = new EventStream(this.#http, `/v1/quality-checks/${this.id}/events`, verbose);
+        const deadline = Date.now() + timeout * 1000;
+        while (this.status === "running") {
+            if (Date.now() > deadline) {
+                throw new Error(`quality check ${this.id} still running after ${timeout}s`);
+            }
+            await sleep(pollInterval * 1000);
+            const data = await this.#http.get(`/v1/quality-checks/${this.id}`);
+            this.status = data.status;
+            this.error = data.error;
+            await events.pump();
+        }
+        await events.pump(); // catch events written after the final status poll
+        if (this.status !== "succeeded") {
+            throw new Error(`quality check ${this.id} failed: ${this.error}`);
+        }
+        return this;
+    }
+    /**
+     * Per-rollout results: rollout_id, passed, states (the agentic-state
+     * trajectory), and judge (dimensions + issues).
+     */
+    async rollouts() {
+        const body = await this.#http.get(`/v1/quality-checks/${this.id}/rollouts`);
+        return body.data;
+    }
+}
+export class Rollouts {
+    #http;
+    #sessionId;
+    constructor(http, sessionId = null) {
+        this.#http = http;
+        this.#sessionId = sessionId;
+    }
+    /**
+     * A stored rollout's full captured state: status, seed, transcript,
+     * tool events, and sandbox.
+     */
+    async get(rolloutId) {
+        return this.#http.get(`/v1/rollouts/${rolloutId}`);
+    }
+    /**
+     * Play a dataset's scenarios against `agent` (most recent dataset when
+     * none is given).
+     *
+     * The agent runs locally: each turn it gets the transcript so far and
+     * a deterministic ToolSandbox for its tool calls; only its reply and
+     * tool events travel over the wire. Scenarios run with `concurrency`
+     * in-flight at once (turns within one conversation are sequential) —
+     * pass concurrency: 1 to run them strictly one at a time.
+     */
+    async run(agent, dataset = null, opts = {}) {
+        const { maxTurns = 12, concurrency = 4 } = opts;
+        let datasetId;
+        if (dataset === null) {
+            // Session-scoped default: this script's latest dataset, so two
+            // concurrent scripts never pick up each other's data.
+            let data = [];
+            if (this.#sessionId) {
+                const body = await this.#http.get("/v1/datasets", {
+                    sdk_session: this.#sessionId,
+                });
+                data = body.data;
+            }
+            if (!data.length) {
+                const body = await this.#http.get("/v1/datasets");
+                data = body.data;
+                if (data.length && this.#sessionId) {
+                    console.log(`note: no dataset in this session yet; ` +
+                        `using latest dataset ${data[0].id}`);
+                }
+            }
+            if (!data.length) {
+                throw new Error("no datasets exist yet; generate one first");
+            }
+            datasetId = data[0].id;
+        }
+        else {
+            datasetId = typeof dataset === "string" ? dataset : dataset.id;
+        }
+        const body = await this.#http.get(`/v1/datasets/${datasetId}/rows`);
+        const rows = body.data;
+        const results = new Array(rows.length);
+        let next = 0;
+        const worker = async () => {
+            while (next < rows.length) {
+                const i = next++;
+                results[i] = await this.runScenario(agent, rows[i].scenario_id, {
+                    maxTurns,
+                });
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(concurrency, rows.length) }, worker));
+        return results;
+    }
+    /**
+     * Start an async quality check over finished rollouts: the server
+     * analyzes each rollout's agentic states in parallel and judges
+     * whether the agent passed each scenario.
+     */
+    async qualityCheck(rollouts) {
+        const rolloutIds = rollouts.map((r) => typeof r === "string" ? r : r.rollout_id);
+        const data = await this.#http.post("/v1/quality-checks", {
+            rollout_ids: rolloutIds,
+        });
+        return new QualityCheck(data, this.#http);
+    }
+    /** Run one rollout session; one HTTP round-trip per agent turn. */
+    async runScenario(agent, scenarioId, opts = {}) {
+        const { maxTurns = 12, randomSeed = null } = opts;
+        let session = await this.#http.post("/v1/rollouts", {
+            scenario_id: scenarioId,
+            random_seed: randomSeed,
+            max_turns: maxTurns,
+        });
+        while (session.status === "running") {
+            const sandbox = ToolSandbox.fromConfig(session.sandbox);
+            const reply = await agent(session.transcript, sandbox);
+            session = await this.#http.post(`/v1/rollouts/${session.id}/turns`, {
+                reply,
+                tool_calls: sandbox.events,
+            });
+        }
+        return {
+            rollout_id: session.id,
+            scenario_id: scenarioId,
+            status: session.status,
+            turns: session.turn,
+            transcript: session.transcript,
+            tool_events: session.tool_events,
+        };
+    }
+}
+/**
+ * Client entry point.
+ *
+ * Session identity: every client belongs to a named session — the stable,
+ * account-scoped identity of one script, persisted across executions
+ * (same name resumes the same session; re-runs reuse its datasets instead
+ * of re-probing/re-generating). Resolution order: `session` option >
+ * SYNTHIA_SESSION env var > derived "project/script" name from the entry
+ * point. `session: false` opts out into a fresh ephemeral session.
+ *
+ * Degradation: an old server without /v1/sdk-sessions -> no session;
+ * keyless against a keyed server -> anonymous session; an invalid apiKey
+ * fails on first use with the server's message.
+ */
+export class Synthia {
+    sessionName;
+    sessionId = null;
+    invocationId = null;
+    seeds;
+    userModels;
+    datasets;
+    rollouts;
+    #http;
+    constructor(options = {}) {
+        const apiKey = options.apiKey ?? process.env["SYNTHIA_API_KEY"];
+        const baseUrl = options.baseUrl ?? process.env["SYNTHIA_BASE_URL"] ?? DEFAULT_BASE_URL;
+        const headers = apiKey
+            ? { authorization: `Bearer ${apiKey}` }
+            : {};
+        this.#http = new Http(baseUrl, headers);
+        if (options.session === false) {
+            this.sessionName =
+                `${defaultSessionName()}/eph-` +
+                    randomUUID().replace(/-/g, "").slice(0, 8);
+        }
+        else if (options.session) {
+            this.sessionName = options.session;
+        }
+        else {
+            this.sessionName =
+                process.env["SYNTHIA_SESSION"] || defaultSessionName();
+        }
+        // The handshake gates every later request, so all of them carry the
+        // session headers without callers having to await anything up front.
+        this.#http.ready = this.#startSession();
+        this.seeds = new Seeds(this.#http);
+        this.userModels = new UserModels(this.#http);
+        this.datasets = new Datasets(this.#http);
+        this.rollouts = new Rollouts(this.#http, null);
+    }
+    /**
+     * One handshake per process: get-or-create the named session and mint
+     * this invocation; all later requests carry both ids as headers.
+     */
+    async #startSession() {
+        let r;
+        try {
+            r = await this.#http.raw("POST", "/v1/sdk-sessions", {
+                name: this.sessionName,
+                sdk_version: SDK_VERSION,
+            });
+        }
+        catch {
+            return; // transient network trouble: run without session tracking
+        }
+        if (r.status === 404)
+            return; // server predates sessions: degrade
+        if (r.status === 401) {
+            const detail = await r
+                .json()
+                .then((b) => b.detail)
+                .catch(() => null);
+            throw new Error(detail ?? "invalid API key");
+        }
+        if (!r.ok) {
+            throw new HttpError(r.status, await r.text(), "POST", "/v1/sdk-sessions");
+        }
+        const data = (await r.json());
+        this.sessionId = data.sdk_session_id;
+        this.invocationId = data.sdk_invocation_id;
+        this.#http.headers["X-Synthia-Session"] = this.sessionId;
+        this.#http.headers["X-Synthia-Invocation"] = this.invocationId;
+        this.rollouts = new Rollouts(this.#http, this.sessionId);
+    }
+    /**
+     * Probe + generate only when needed; otherwise reuse the latest dataset.
+     *
+     * The main entry point for the probe and generation steps. `count` is
+     * exact: the returned dataset has exactly that many rows, so reuse
+     * requires the latest dataset to match it in addition to the quality
+     * gate. Probing and generation run only when no dataset exists yet,
+     * when the row count differs (generation-only — the session's probed
+     * user model is reused), or when the latest completed quality check's
+     * pass rate falls outside [minSuccessRate, maxSuccessRate].
+     * Out-of-band regeneration passes that quality check to the server,
+     * which feeds its real results into scenario generation so the new
+     * batch recalibrates difficulty and coverage.
+     *
+     * All lookups are scoped to this client's session: re-running the same
+     * script reuses its own dataset, and drift signals from other
+     * scripts/sessions never trigger regeneration here.
+     */
+    async prepare(agent, opts = {}) {
+        const { count = 20, maxTurns = 10, minSuccessRate = 0.6, maxSuccessRate = 0.9, verbose = false, } = opts;
+        await this.#http.ready;
+        const existing = await this.datasets.list(this.sessionId ?? undefined); // newest first
+        if (!existing.length) {
+            return this.#probeAndGenerate(agent, {
+                count,
+                maxTurns,
+                verbose,
+                reason: this.sessionId
+                    ? "no datasets in this session yet"
+                    : "no datasets exist yet",
+                successRate: null,
+                qualityCheckId: null,
+            });
+        }
+        const latest = await this.#http.get("/v1/quality-checks/latest", this.sessionId ? { sdk_session: this.sessionId } : undefined);
+        const rate = latest.id !== null && latest.total > 0
+            ? latest.passed / latest.total
+            : null;
+        if (rate !== null && !(minSuccessRate <= rate && rate <= maxSuccessRate)) {
+            const direction = rate < minSuccessRate ? "below" : "above";
+            const bound = rate < minSuccessRate ? minSuccessRate : maxSuccessRate;
+            return this.#probeAndGenerate(agent, {
+                count,
+                maxTurns,
+                verbose,
+                reason: `success rate ${pct(rate)} ${direction} ${pct(bound)}; ` +
+                    `regenerating calibrated on ${latest.id}`,
+                successRate: rate,
+                qualityCheckId: latest.id,
+            });
+        }
+        // Quality is in band (or unjudged): reuse only on an exact size
+        // match; otherwise regenerate at the requested count — without
+        // re-probing, since nothing suggests the agent changed.
+        if (existing[0].row_count !== count) {
+            return this.#probeAndGenerate(agent, {
+                count,
+                maxTurns,
+                verbose,
+                reason: `latest dataset has ${existing[0].row_count} rows; ` +
+                    `requested ${count}`,
+                successRate: rate,
+                qualityCheckId: null,
+            });
+        }
+        return {
+            dataset: existing[0],
+            userModel: await this.userModels.get(existing[0].user_model_id),
+            action: "reused",
+            reason: rate !== null
+                ? `success rate ${pct(rate)} within ` +
+                    `${pct(minSuccessRate)}-${pct(maxSuccessRate)} band`
+                : "no completed quality check to judge by; reusing latest dataset",
+            successRate: rate,
+            qualityCheckId: null,
+        };
+    }
+    async #probeAndGenerate(agent, opts) {
+        // Without a drift signal the agent hasn't been shown to change, so a
+        // user model this session already probed is still good — skip the
+        // probe. Drift-triggered regeneration re-probes deliberately.
+        let reason = opts.reason;
+        let userModel = null;
+        if (this.sessionId && !opts.qualityCheckId) {
+            const sessionModels = await this.userModels.list(this.sessionId);
+            if (sessionModels.length) {
+                userModel = sessionModels[sessionModels.length - 1]; // newest last
+                reason += "; reusing session user model (no drift signal)";
+            }
+        }
+        if (userModel === null) {
+            userModel = await this.userModels.createFromProbe(agent, {
+                maxTurns: opts.maxTurns,
+                verbose: opts.verbose,
+            });
+        }
+        const job = await this.datasets.generate(userModel, {
+            count: opts.count,
+            qualityCheckId: opts.qualityCheckId ?? undefined,
+        });
+        const dataset = await job.wait({ verbose: opts.verbose });
+        return {
+            dataset,
+            userModel,
+            action: "generated",
+            reason,
+            successRate: opts.successRate,
+            qualityCheckId: opts.qualityCheckId,
+        };
+    }
+}
+const pct = (x) => `${Math.round(x * 100)}%`;

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export { Synthia, ToolSandbox, Dataset, GenerationJob, ValidationRun, QualityCheck, Seeds, UserModels, Datasets, Rollouts, DEFAULT_BASE_URL, } from "./client.js";
+export type { Agent, AgentReply, PrepareOptions, PrepareResult, RolloutAgent, RolloutResult, RunOptions, SandboxConfig, SynthiaOptions, ToolCall, ToolEvent, TranscriptTurn, UserModel, WaitOptions, } from "./client.js";

package/dist/index.js

@@ -0,0 +1 @@
+export { Synthia, ToolSandbox, Dataset, GenerationJob, ValidationRun, QualityCheck, Seeds, UserModels, Datasets, Rollouts, DEFAULT_BASE_URL, } from "./client.js";

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "synthiaresearch",
+  "version": "0.0.1",
+  "description": "JavaScript/TypeScript SDK for the Synthia API",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "build": "tsc",
+    "prepublishOnly": "npm run build"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "typescript": "^5.5.0"
+  }
+}