xo-harness 0.1.0

package/README.md

@@ -0,0 +1,32 @@
+# xo-harness
+
+`xo-harness` is a TypeScript-first agent harness for continuous, fully duplex voice models. One package
+contains the harness, provider adapters, durable stores, shared protocol, and deterministic testing
+providers.
+
+```sh
+npm install xo-harness
+```
+
+```ts
+import { XO } from "xo-harness/harness";
+import { OpenAIRealtimeVoiceProvider } from "xo-harness/provider";
+import { JsonlEventStore } from "xo-harness/storage";
+
+const harness = new XO({ store: new JsonlEventStore(".xo/sessions") });
+const session = await harness.startSession({
+  provider: new OpenAIRealtimeVoiceProvider({ apiKey: process.env.OPENAI_API_KEY! }),
+});
+```
+
+Additional entry points are available for focused imports:
+
+- `xo-harness/protocol`
+- `xo-harness/harness`
+- `xo-harness/provider`
+- `xo-harness/provider/workers`
+- `xo-harness/storage`
+- `xo-harness/testing`
+
+See the [XO repository](https://github.com/zvadaadam/XO) for the complete documentation and voice
+application.

package/dist/harness.d.ts

@@ -0,0 +1 @@
+export * from "./internal/harness/index.js";

package/dist/harness.js

@@ -0,0 +1 @@
+export * from "./internal/harness/index.js";

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./internal/harness/index.js";
+export * from "./internal/protocol/index.js";
+export * from "./internal/storage/index.js";
+export { z } from "zod";

package/dist/index.js

@@ -0,0 +1,4 @@
+export * from "./internal/harness/index.js";
+export * from "./internal/protocol/index.js";
+export * from "./internal/storage/index.js";
+export { z } from "zod";

package/dist/internal/harness/event-stream.d.ts

@@ -0,0 +1,7 @@
+import { type HarnessEvent } from "../protocol/index.js";
+export declare class ReplayEventStream {
+    #private;
+    publish(event: HarnessEvent): void;
+    subscribe(afterSequence?: number): AsyncIterable<HarnessEvent>;
+    close(): void;
+}

package/dist/internal/harness/event-stream.js

@@ -0,0 +1,42 @@
+import { AsyncQueue } from "../protocol/index.js";
+export class ReplayEventStream {
+    #history = [];
+    #subscribers = new Set();
+    #closed = false;
+    publish(event) {
+        if (this.#closed)
+            throw new Error("Cannot publish after the event stream is closed");
+        this.#history.push(event);
+        for (const subscriber of this.#subscribers)
+            subscriber.push(event);
+    }
+    subscribe(afterSequence = 0) {
+        const queue = new AsyncQueue();
+        const backlog = this.#history.filter((event) => event.sequence > afterSequence);
+        for (const event of backlog)
+            queue.push(event);
+        if (this.#closed)
+            queue.close();
+        else
+            this.#subscribers.add(queue);
+        const subscribers = this.#subscribers;
+        return {
+            async *[Symbol.asyncIterator]() {
+                try {
+                    yield* queue;
+                }
+                finally {
+                    subscribers.delete(queue);
+                }
+            },
+        };
+    }
+    close() {
+        if (this.#closed)
+            return;
+        this.#closed = true;
+        for (const subscriber of this.#subscribers)
+            subscriber.close();
+        this.#subscribers.clear();
+    }
+}

package/dist/internal/harness/index.d.ts

@@ -0,0 +1,7 @@
+export * from "./message.js";
+export * from "./report.js";
+export * from "./shadow.js";
+export * from "./socket-bridge.js";
+export * from "./tools.js";
+export * from "./voice-session.js";
+export * from "./xo.js";

package/dist/internal/harness/index.js

@@ -0,0 +1,7 @@
+export * from "./message.js";
+export * from "./report.js";
+export * from "./shadow.js";
+export * from "./socket-bridge.js";
+export * from "./tools.js";
+export * from "./voice-session.js";
+export * from "./xo.js";

package/dist/internal/harness/message.d.ts

@@ -0,0 +1,26 @@
+import type { HarnessEvent, Message } from "../protocol/index.js";
+/**
+ * Projects the flat, sequence-ordered event log into the v2 `Message[]` part model —
+ * the shape every agent harness renders and persists (opencode / Vercel AI SDK v5 /
+ * agent-server / echo). Pure AND deterministic over the recorded events: ids derive
+ * from the opening event, so the same log always projects to the same identities —
+ * re-folds diff cleanly, upserts stay idempotent, and renderers can key on part ids.
+ *
+ * Folding rules:
+ *  - `message.input` → a user message; its recorded `parts` project with full fidelity
+ *    (image/file parts become `file` parts), a bare-text input becomes one text part.
+ *  - user `transcript` → a user message with one text part.
+ *  - assistant `transcript.delta` → a `streaming` text part accumulating deltas,
+ *    finalized in place by the matching `transcript` (streamId-correlated, so a
+ *    barge-in interrupted stream still finalizes into its original part).
+ *  - `audio.output` → an `audio` part on the current assistant message (deduped per
+ *    streamId), linked to its transcript text part by that streamId.
+ *  - `tool.requested` → ONE `tool` part in `running` state; `tool.completed`/`tool.failed`
+ *    settle it with the canonical outcome, and a task terminal upgrades an
+ *    `accepted_task` outcome to the task's real result. This is why a tool result
+ *    (e.g. a crisis card) replays from history: the outcome lives on the part.
+ *
+ * A tool part belongs to the assistant message open when it was requested; a new
+ * assistant transcript or user turn opens the next message.
+ */
+export declare function projectMessages(events: readonly HarnessEvent[]): Message[];

package/dist/internal/harness/message.js

@@ -0,0 +1,192 @@
+/**
+ * Projects the flat, sequence-ordered event log into the v2 `Message[]` part model —
+ * the shape every agent harness renders and persists (opencode / Vercel AI SDK v5 /
+ * agent-server / echo). Pure AND deterministic over the recorded events: ids derive
+ * from the opening event, so the same log always projects to the same identities —
+ * re-folds diff cleanly, upserts stay idempotent, and renderers can key on part ids.
+ *
+ * Folding rules:
+ *  - `message.input` → a user message; its recorded `parts` project with full fidelity
+ *    (image/file parts become `file` parts), a bare-text input becomes one text part.
+ *  - user `transcript` → a user message with one text part.
+ *  - assistant `transcript.delta` → a `streaming` text part accumulating deltas,
+ *    finalized in place by the matching `transcript` (streamId-correlated, so a
+ *    barge-in interrupted stream still finalizes into its original part).
+ *  - `audio.output` → an `audio` part on the current assistant message (deduped per
+ *    streamId), linked to its transcript text part by that streamId.
+ *  - `tool.requested` → ONE `tool` part in `running` state; `tool.completed`/`tool.failed`
+ *    settle it with the canonical outcome, and a task terminal upgrades an
+ *    `accepted_task` outcome to the task's real result. This is why a tool result
+ *    (e.g. a crisis card) replays from history: the outcome lives on the part.
+ *
+ * A tool part belongs to the assistant message open when it was requested; a new
+ * assistant transcript or user turn opens the next message.
+ */
+export function projectMessages(events) {
+    const started = events.find((event) => event.type === "session.started");
+    const sessionId = started?.sessionId ?? events[0]?.sessionId ?? "unknown";
+    const messages = [];
+    let current;
+    // callId → the tool part being advanced, so settlement updates the same part.
+    const toolParts = new Map();
+    // streamId → the assistant message + audio part, so repeated chunks fold into one.
+    const audioParts = new Map();
+    // stream key → the streaming text part accumulating deltas, finalized by `transcript`.
+    // Deltas without a streamId key on the open assistant message, so a new message
+    // naturally starts a new default stream — no sentinel, no clearing.
+    const textParts = new Map();
+    // Identity is the opening event: message id = event id, part id = event id + ordinal.
+    const partBase = (message, event, ordinal = 0) => ({
+        id: `${event.id}/p${ordinal}`,
+        sessionId,
+        messageId: message.id,
+    });
+    const openMessage = (role, event) => {
+        current = { id: event.id, sessionId, role, createdAtMs: event.recordedAtMs, parts: [] };
+        messages.push(current);
+        return current;
+    };
+    const assistantMessage = (event) => {
+        if (current?.role === "assistant")
+            return current;
+        return openMessage("assistant", event);
+    };
+    for (const event of events) {
+        switch (event.type) {
+            case "message.input": {
+                const message = openMessage("user", event);
+                for (const [ordinal, part] of (event.parts ?? [{ type: "text", text: event.text }]).entries()) {
+                    message.parts.push(part.type === "text"
+                        ? { ...partBase(message, event, ordinal), type: "text", text: part.text, state: "done" }
+                        : {
+                            ...partBase(message, event, ordinal),
+                            type: "file",
+                            mediaType: part.mediaType,
+                            url: part.url,
+                            ...(part.type === "file" && part.filename !== undefined ? { filename: part.filename } : {}),
+                        });
+                }
+                break;
+            }
+            case "transcript.delta": {
+                if (event.role !== "assistant")
+                    break;
+                const message = assistantMessage(event);
+                const key = event.streamId ?? message.id;
+                const existing = textParts.get(key);
+                if (existing) {
+                    existing.text += event.delta;
+                    break;
+                }
+                const part = {
+                    ...partBase(message, event),
+                    type: "text",
+                    text: event.delta,
+                    state: "streaming",
+                };
+                textParts.set(key, part);
+                message.parts.push(part);
+                break;
+            }
+            case "transcript": {
+                if (event.role === "user") {
+                    const message = openMessage("user", event);
+                    message.parts.push({ ...partBase(message, event), type: "text", text: event.text, state: "done" });
+                    break;
+                }
+                // Finalize the streaming part these deltas were building, if any — even when a
+                // barge-in already moved the conversation on — else record a fresh done part.
+                const key = event.streamId ?? (current?.role === "assistant" ? current.id : undefined);
+                const streaming = key === undefined ? undefined : textParts.get(key);
+                if (key !== undefined && streaming !== undefined) {
+                    streaming.text = event.text;
+                    streaming.state = "done";
+                    textParts.delete(key);
+                    break;
+                }
+                const message = assistantMessage(event);
+                message.parts.push({ ...partBase(message, event), type: "text", text: event.text, state: "done" });
+                break;
+            }
+            case "audio.output": {
+                const streamId = event.chunk.streamId;
+                if (audioParts.has(streamId))
+                    break;
+                const message = assistantMessage(event);
+                const part = {
+                    ...partBase(message, event),
+                    type: "audio",
+                    streamId,
+                    sampleRate: event.chunk.sampleRate,
+                    state: "done",
+                };
+                audioParts.set(streamId, part);
+                message.parts.push(part);
+                break;
+            }
+            case "tool.requested": {
+                if (toolParts.has(event.call.callId))
+                    break;
+                const message = assistantMessage(event);
+                const part = {
+                    ...partBase(message, event),
+                    type: "tool",
+                    toolCallId: event.call.callId,
+                    toolName: event.call.name,
+                    state: { status: "running", input: event.call.arguments, startedAtMs: event.recordedAtMs },
+                };
+                toolParts.set(event.call.callId, part);
+                message.parts.push(part);
+                break;
+            }
+            case "tool.completed": {
+                const part = toolParts.get(event.callId);
+                if (part?.state.status !== "running")
+                    break;
+                part.state = {
+                    status: "settled",
+                    input: part.state.input,
+                    outcome: event.outcome,
+                    startedAtMs: part.state.startedAtMs,
+                    endedAtMs: event.recordedAtMs,
+                };
+                break;
+            }
+            case "tool.failed": {
+                const part = toolParts.get(event.callId);
+                if (part?.state.status !== "running")
+                    break;
+                part.state = {
+                    status: "settled",
+                    input: part.state.input,
+                    outcome: { type: "failed", error: event.error },
+                    startedAtMs: part.state.startedAtMs,
+                    endedAtMs: event.recordedAtMs,
+                };
+                break;
+            }
+            case "task.completed":
+            case "task.failed":
+            case "task.cancelled": {
+                // A background task settling upgrades its tool part's accepted_task handoff to
+                // the real result, so task-shaped tools replay from history like direct ones.
+                const part = toolParts.get(event.callId);
+                if (part?.state.status !== "settled" || part.state.outcome.type !== "accepted_task")
+                    break;
+                part.state = {
+                    ...part.state,
+                    outcome: event.type === "task.completed"
+                        ? { type: "completed", value: event.result }
+                        : event.type === "task.failed"
+                            ? { type: "failed", error: event.error }
+                            : { type: "failed", error: `Task cancelled: ${event.reason}` },
+                    endedAtMs: event.recordedAtMs,
+                };
+                break;
+            }
+            default:
+                break;
+        }
+    }
+    return messages;
+}

package/dist/internal/harness/report.d.ts

@@ -0,0 +1,51 @@
+import { type HarnessEvent, type ToolOutcome } from "../protocol/index.js";
+export interface ToolCallReport {
+    callId: string;
+    name: string;
+    arguments: Record<string, unknown>;
+    requestedAtMs: number;
+    outcome?: ToolOutcome;
+    latencyMs?: number;
+}
+export interface TranscriptLine {
+    role: "user" | "assistant";
+    text: string;
+    atMs: number;
+}
+/** Session token totals summed over provider usage reports; zeros when none arrived. */
+export interface UsageTotals {
+    responses: number;
+    inputTokens: number;
+    outputTokens: number;
+    totalTokens: number;
+    cachedInputTokens: number;
+    audioInputTokens: number;
+    audioOutputTokens: number;
+    cachedAudioInputTokens: number;
+}
+export interface SessionReport {
+    sessionId: string;
+    providerId: string;
+    durationMs: number;
+    endedReason?: string;
+    readyAtMs?: number;
+    firstInputAtMs?: number;
+    firstOutputAtMs?: number;
+    /** First model audio relative to first microphone audio. */
+    responseLatencyMs?: number;
+    inputAudioMs: number;
+    outputAudioMs: number;
+    toolCalls: ToolCallReport[];
+    transcripts: TranscriptLine[];
+    taskIds: string[];
+    errors: {
+        message: string;
+        recoverable: boolean;
+    }[];
+    usage: UsageTotals;
+}
+/**
+ * Derives latency and outcome metrics from a session log. Pure over the recorded events,
+ * so it works identically on live sessions, replays, and shadow runs.
+ */
+export declare function summarizeSession(events: readonly HarnessEvent[]): SessionReport;

package/dist/internal/harness/report.js

@@ -0,0 +1,115 @@
+import { audioSampleCount } from "../protocol/index.js";
+/**
+ * Derives latency and outcome metrics from a session log. Pure over the recorded events,
+ * so it works identically on live sessions, replays, and shadow runs.
+ */
+export function summarizeSession(events) {
+    const started = events.find((event) => event.type === "session.started");
+    if (!started)
+        throw new Error("Session log has no session.started event");
+    let endedReason;
+    let readyAtMs;
+    let firstInputAtMs;
+    let firstOutputAtMs;
+    let inputAudioMs = 0;
+    let outputAudioMs = 0;
+    const toolCalls = new Map();
+    const transcripts = [];
+    const taskIds = new Set();
+    const errors = [];
+    const usage = {
+        responses: 0,
+        inputTokens: 0,
+        outputTokens: 0,
+        totalTokens: 0,
+        cachedInputTokens: 0,
+        audioInputTokens: 0,
+        audioOutputTokens: 0,
+        cachedAudioInputTokens: 0,
+    };
+    for (const event of events) {
+        switch (event.type) {
+            case "provider.ready":
+                readyAtMs ??= event.sessionTimeMs;
+                break;
+            case "usage":
+                usage.responses += 1;
+                usage.inputTokens += event.inputTokens;
+                usage.outputTokens += event.outputTokens;
+                usage.totalTokens += event.totalTokens;
+                usage.cachedInputTokens += event.cachedInputTokens ?? 0;
+                usage.audioInputTokens += event.audioInputTokens ?? 0;
+                usage.audioOutputTokens += event.audioOutputTokens ?? 0;
+                usage.cachedAudioInputTokens += event.cachedAudioInputTokens ?? 0;
+                break;
+            case "audio.input":
+                firstInputAtMs ??= event.sessionTimeMs;
+                inputAudioMs += (audioSampleCount(event.chunk) / event.chunk.sampleRate) * 1000;
+                break;
+            case "audio.output":
+                firstOutputAtMs ??= event.sessionTimeMs;
+                outputAudioMs += (audioSampleCount(event.chunk) / event.chunk.sampleRate) * 1000;
+                break;
+            case "transcript":
+                transcripts.push({ role: event.role, text: event.text, atMs: event.sessionTimeMs });
+                break;
+            case "message.input":
+                transcripts.push({ role: "user", text: event.text, atMs: event.sessionTimeMs });
+                break;
+            case "tool.requested":
+                toolCalls.set(event.call.callId, {
+                    callId: event.call.callId,
+                    name: event.call.name,
+                    arguments: event.call.arguments,
+                    requestedAtMs: event.sessionTimeMs,
+                });
+                break;
+            case "tool.completed": {
+                const call = toolCalls.get(event.callId);
+                if (call) {
+                    call.outcome = event.outcome;
+                    call.latencyMs = event.sessionTimeMs - call.requestedAtMs;
+                }
+                break;
+            }
+            case "tool.failed": {
+                const call = toolCalls.get(event.callId);
+                if (call) {
+                    call.outcome = { type: "failed", error: event.error };
+                    call.latencyMs = event.sessionTimeMs - call.requestedAtMs;
+                }
+                break;
+            }
+            case "task.accepted":
+                taskIds.add(event.taskId);
+                break;
+            case "provider.error":
+                errors.push({ message: event.message, recoverable: event.recoverable });
+                break;
+            case "session.ended":
+                endedReason = event.reason;
+                break;
+            default:
+                break;
+        }
+    }
+    return {
+        sessionId: started.sessionId,
+        providerId: started.providerId,
+        durationMs: events.at(-1)?.sessionTimeMs ?? 0,
+        inputAudioMs: Math.round(inputAudioMs),
+        outputAudioMs: Math.round(outputAudioMs),
+        toolCalls: [...toolCalls.values()],
+        transcripts,
+        taskIds: [...taskIds],
+        usage,
+        errors,
+        ...(endedReason === undefined ? {} : { endedReason }),
+        ...(readyAtMs === undefined ? {} : { readyAtMs }),
+        ...(firstInputAtMs === undefined ? {} : { firstInputAtMs }),
+        ...(firstOutputAtMs === undefined ? {} : { firstOutputAtMs }),
+        ...(firstInputAtMs === undefined || firstOutputAtMs === undefined
+            ? {}
+            : { responseLatencyMs: firstOutputAtMs - firstInputAtMs }),
+    };
+}

package/dist/internal/harness/shadow.d.ts

@@ -0,0 +1,28 @@
+import type { HarnessEvent } from "../protocol/index.js";
+import type { VoiceProvider } from "../provider/index.js";
+import { type EventStore } from "../storage/index.js";
+import { type SessionReport } from "./report.js";
+import type { VoiceTool } from "./tools.js";
+export interface ShadowRunOptions {
+    /** The recorded session whose microphone audio is re-driven. */
+    events: readonly HarnessEvent[];
+    /** The provider to shadow-test with the recorded input. */
+    provider: VoiceProvider;
+    /** Tools to expose; usually the same implementations the original session had. */
+    tools?: readonly VoiceTool[];
+    /** Where the shadow session records. Defaults to an in-memory store. */
+    store?: EventStore;
+    sessionId?: string;
+    /** Defaults to the recorded session's instructions. */
+    instructions?: string;
+    /** The run settles once the provider has produced audio and then stayed quiet this long. */
+    quietMs?: number;
+    timeoutMs?: number;
+}
+/**
+ * The silent-test primitive: re-drives a recorded session's input audio against another
+ * provider and returns the resulting session report for comparison with the original.
+ * Input is sent as fast as the provider accepts it; server-side VAD sees the recorded
+ * speech and trailing silence exactly as the original session did.
+ */
+export declare function runShadowSession(options: ShadowRunOptions): Promise<SessionReport>;

package/dist/internal/harness/shadow.js

@@ -0,0 +1,118 @@
+import { MemoryEventStore } from "../storage/index.js";
+import { summarizeSession } from "./report.js";
+import { XO } from "./xo.js";
+/**
+ * The silent-test primitive: re-drives a recorded session's input audio against another
+ * provider and returns the resulting session report for comparison with the original.
+ * Input is sent as fast as the provider accepts it; server-side VAD sees the recorded
+ * speech and trailing silence exactly as the original session did.
+ */
+export async function runShadowSession(options) {
+    const recordedStart = options.events.find((event) => event.type === "session.started");
+    const inputs = options.events.filter((event) => event.type === "audio.input");
+    if (inputs.length === 0)
+        throw new Error("Recorded session has no input audio to re-drive");
+    const instructions = options.instructions ?? recordedStart?.instructions;
+    const quietMs = options.quietMs ?? 3_000;
+    const timeoutMs = options.timeoutMs ?? 60_000;
+    const harness = new XO({
+        store: options.store ?? new MemoryEventStore(),
+        ...(options.tools === undefined ? {} : { tools: options.tools }),
+    });
+    const session = await harness.startSession({
+        provider: options.provider,
+        maxDurationMs: timeoutMs,
+        ...(options.sessionId === undefined ? {} : { sessionId: options.sessionId }),
+        ...(instructions === undefined ? {} : { instructions }),
+    });
+    try {
+        await waitForReady(session, timeoutMs);
+        for (const input of inputs) {
+            await session.sendAudio(input.chunk);
+        }
+        await settleAfterOutput(session, quietMs, timeoutMs);
+    }
+    finally {
+        await session.close("shadow_complete");
+    }
+    return summarizeSession(await session.history());
+}
+function createEventCursor(session) {
+    const iterator = session.events()[Symbol.asyncIterator]();
+    // A raced-and-lost next() must be reused, not abandoned: an abandoned call would
+    // consume the very event the following next() is waiting for.
+    let pendingNext;
+    return {
+        async next(timeoutMs) {
+            pendingNext ??= iterator.next();
+            const result = await Promise.race([pendingNext, delay(timeoutMs).then(() => "timeout")]);
+            if (result === "timeout")
+                return "timeout";
+            pendingNext = undefined;
+            return result.done ? "done" : result.value;
+        },
+        dispose() {
+            // Not awaited: with a next() in flight, an async generator's return() settles only
+            // after that next() does — which here means only once the session closes.
+            void iterator.return?.();
+        },
+    };
+}
+async function waitForReady(session, timeoutMs) {
+    const cursor = createEventCursor(session);
+    const deadline = Date.now() + timeoutMs;
+    try {
+        while (true) {
+            const remaining = deadline - Date.now();
+            if (remaining <= 0)
+                throw new Error("Shadow provider did not become ready in time");
+            const event = await cursor.next(remaining);
+            if (event === "timeout")
+                throw new Error("Shadow provider did not become ready in time");
+            if (event === "done")
+                throw new Error("Shadow session ended before the provider was ready");
+            if (event.type === "provider.ready")
+                return;
+        }
+    }
+    finally {
+        cursor.dispose();
+    }
+}
+async function settleAfterOutput(session, quietMs, timeoutMs) {
+    const cursor = createEventCursor(session);
+    const deadline = Date.now() + timeoutMs;
+    let lastActivity = Date.now();
+    let sawOutput = false;
+    try {
+        while (true) {
+            const untilDeadline = deadline - Date.now();
+            if (untilDeadline <= 0)
+                return;
+            const untilQuiet = quietMs - (Date.now() - lastActivity);
+            if (sawOutput && untilQuiet <= 0)
+                return;
+            const wait = Math.max(10, sawOutput ? Math.min(untilQuiet, untilDeadline) : untilDeadline);
+            const event = await cursor.next(wait);
+            if (event === "timeout")
+                continue;
+            if (event === "done")
+                return;
+            const type = event.type;
+            if (type === "audio.output" ||
+                type === "transcript" ||
+                type === "tool.requested" ||
+                type === "tool.completed") {
+                lastActivity = Date.now();
+                if (type === "audio.output")
+                    sawOutput = true;
+            }
+        }
+    }
+    finally {
+        cursor.dispose();
+    }
+}
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}