xo-harness 0.2.1 → 0.3.0

package/README.md

@@ -4,28 +4,78 @@
 contains the harness, provider adapters, durable stores, shared protocol, and deterministic testing
 providers.
 
+GPT Live is an opt-in adapter in 0.3.0. Real tests found occasional duplicate typed greetings and
+incorrect spoken associations despite correct tool execution; captions are not proof of playback.
+Keep verified tool outcomes authoritative and evaluate your application's prompts and audio path.
+See the repository's [production evidence and limits](https://github.com/zvadaadam/XO/blob/main/docs/production-readiness.md).
+
 ```sh
 npm install xo-harness
 ```
 
+This Node smoke test opens Realtime, observes one assistant response, and closes the session.
+The explicit limits are starting values to measure for your application, not a production preset.
+
 ```ts
 import { XO } from "xo-harness/harness";
 import { OpenAIRealtimeVoiceProvider } from "xo-harness/provider";
 import { JsonlEventStore } from "xo-harness/storage";
 
-const harness = new XO({ store: new JsonlEventStore(".xo/sessions") });
+const apiKey = process.env.OPENAI_API_KEY;
+if (!apiKey) throw new Error("Set OPENAI_API_KEY");
+const harness = new XO({
+  // Keep conversation evidence and audio timing, without storing a recording.
+  store: new JsonlEventStore(".xo/sessions", { audioRetention: "metadata" }),
+  runtimeLimits: {
+    maxReplayEvents: 2_000,
+    maxReplayAudioBytes: 2 * 1024 * 1024,
+    maxPendingAppends: 64,
+    maxPendingAudioBytes: 512 * 1024,
+    persistenceTimeoutMs: 2_000,
+  },
+});
 const session = await harness.startSession({
-  provider: new OpenAIRealtimeVoiceProvider({ apiKey: process.env.OPENAI_API_KEY! }),
+  provider: new OpenAIRealtimeVoiceProvider({
+    apiKey,
+    eventQueueLimits: { maxEvents: 512, maxAudioBytes: 1_440_000 },
+  }),
+  maxDurationMs: 30_000,
 });
 
-// The model can open from its existing instructions/context; no fake user turn is added.
-await session.requestAssistantTurn({ instructions: "Welcome the caller in one sentence." });
+try {
+  await session.requestAssistantTurn({ instructions: "Welcome the caller in one sentence." });
+  let receivedAssistantTranscript = false;
+  for await (const event of session.events()) {
+    console.log(event.sequence, event.type);
+    if (event.type === "transcript" && event.role === "assistant") receivedAssistantTranscript = true;
+    // Realtime's final response/usage follows its final transcript.
+    if (receivedAssistantTranscript && event.type === "response.state" && event.state === "idle") break;
+  }
+} finally {
+  await session.close("example_finished");
+}
 ```
 
+This example observes generated output; it does not open a microphone or play audio. For a real
+client, use the [browser transport and complete application lifecycle](https://github.com/zvadaadam/XO/blob/main/docs/integrating.md).
+GPT Live uses continuous audio: enable forwarding, then append opening instructions after the first
+input frame. Wait for their matching `context.delivery` acknowledgement, check the opening signal,
+and append the commentary cue “Begin the conversation now, following the instructions provided.”
+Keep input audio running while waiting; `appendContext()` itself confirms submission only. See the
+integration guide for failure and cancellation handling. This differs from Realtime's `requestAssistantTurn`.
+
+Library defaults preserve full audio storage and unlimited replay/storage waits for compatibility.
+Metadata retention and runtime limits are independent: omitting recordings does not bound live
+replay. Lagging readers fail explicitly and require source recovery; metadata is not playable audio.
+See [runtime limits](https://github.com/zvadaadam/XO/blob/main/docs/runtime-limits.md) before hosting
+long calls or allowing concurrent draining sessions.
+
 Additional entry points are available for focused imports:
 
 - `xo-harness/protocol`
 - `xo-harness/harness`
+- `xo-harness/browser` (optional microphone/playback client; explicit socket and worklet URLs)
+- `xo-harness/browser/worklets/capture-processor.js` and `.../playback-processor.js` (serve these assets)
 - `xo-harness/provider`
 - `xo-harness/provider/workers`
 - `xo-harness/storage`
@@ -33,7 +83,13 @@ Additional entry points are available for focused imports:
 - `xo-harness/skills` (catalogs, bundled skills, and on-demand reading tools)
 - `xo-harness/skills/node` (filesystem discovery)
 - `xo-harness/tools/openai` (web search and background reasoning with `createOpenAITools`)
-- `xo-harness/testing`
+- `xo-harness/testing` (fake/scripted/replay providers, `waitForEvent`, `runTrajectory`; portable)
+
+For Cloudflare Workers, use `xo-harness/provider/workers` and `xo-harness/storage/memory`, or supply
+a durable `EventStore` for your database. The root entry and `xo-harness/storage` include Node JSONL
+storage; use focused imports in portable code. `MemoryEventStore({ audioRetention: "metadata" })`
+stores no PCM but remains volatile and retains growing metadata. For portable tool modules, declare
+`zod` as a dependency and import `z` from it rather than from the Node-oriented package root.
 
 Skills follow the standard `SKILL.md` format. Compose `createSkillTools(skills)` with your application
 tools; reads use the existing admission policy and durable event log. See the repository's
@@ -48,5 +104,36 @@ work on session close. Voice and worker reasoning efforts are independent. See t
 Tool results are delivered in full by default. `maxToolResultBytes` opts into string previews for
 oversized results; finite values must be integers of at least 128 bytes, excluding provider envelopes.
 
+The harness exposes frozen `session.capabilities`, input submission receipts, targeted
+`session.cancelTask`, and opt-in `tasks.startOutcome` for authoritative worker outcomes. The
+`ConversationProjector` adds ordered persistence changes around the existing message fold;
+`SessionUsageTracker` shares accounting with reports. Applications own database transactions,
+durable worker jobs, memory extraction, and billing.
+
+To continue in a new session, replay stored `HarnessRecord`s through `ConversationProjector`, then
+use `selectConversationContext(snapshot, { maxMessages, maxTextBytes, unconfirmedText })`. It preserves
+roles, text and provenance and reports omitted parts; map its selected text to the provider's startup
+input. Live captions require an explicit choice about unconfirmed text. Byte budgets are not token
+limits, and selecting history neither restores provider state nor reruns tools. See the
+[executable restart example](https://github.com/zvadaadam/XO/blob/main/examples/integration/README.md).
+
+`sendText` supports strings/text-only parts; images and files must use the application backend.
+Browser callers must wait for `onStarted` and catch synchronous submission errors. See the
+[integration guide](https://github.com/zvadaadam/XO/blob/main/docs/integrating.md) for startup signals,
+graceful close, receipts, and migration details.
+
+Before exposing a `VoiceSocketBridge` to clients, supply a product event allowlist through
+`shouldSendEvent`. Without a predicate, JSON events include tool payloads and startup instructions.
+Filtering leaves the durable log, server observers, audio and bridge controls intact. Authenticate
+the caller and select permitted tools/instructions on the server; the SDK does not choose that policy.
+
+For a simple app, register ordinary memory/search functions directly. The optional
+`createOpenAITools({ apiKey, tools: ["web_search"] })` supplies only OpenAI-backed search, without a
+background worker. Voice adapters and the core harness do not depend on that tool bundle.
+
+Test real application tools through `runTrajectory` and `waitForEvent` instead of hand-building
+execution contexts. The [SDK testing guide](https://github.com/zvadaadam/XO/blob/main/docs/testing.md)
+shows the portable helpers and the simple voice/optional worker contract scenarios.
+
 See the [XO repository](https://github.com/zvadaadam/XO) for the complete documentation and voice
 application.

package/dist/browser/worklets/capture-processor.js

@@ -0,0 +1,34 @@
+// Captures mono microphone audio and posts 100 ms Int16 PCM frames to the main thread.
+const FRAME_SAMPLES = 2400;
+
+class CaptureProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.buffer = new Float32Array(FRAME_SAMPLES);
+    this.filled = 0;
+  }
+
+  process(inputs) {
+    const channel = inputs[0]?.[0];
+    if (!channel) return true;
+    let offset = 0;
+    while (offset < channel.length) {
+      const take = Math.min(channel.length - offset, FRAME_SAMPLES - this.filled);
+      this.buffer.set(channel.subarray(offset, offset + take), this.filled);
+      this.filled += take;
+      offset += take;
+      if (this.filled === FRAME_SAMPLES) {
+        const pcm = new Int16Array(FRAME_SAMPLES);
+        for (let i = 0; i < FRAME_SAMPLES; i += 1) {
+          const sample = Math.max(-1, Math.min(1, this.buffer[i]));
+          pcm[i] = Math.round(sample * 32767);
+        }
+        this.port.postMessage(pcm.buffer, [pcm.buffer]);
+        this.filled = 0;
+      }
+    }
+    return true;
+  }
+}
+
+registerProcessor("xo-capture", CaptureProcessor);

package/dist/browser/worklets/playback-processor.js

@@ -0,0 +1,188 @@
+// Plays queued PCM16 model audio and reports played-through sample positions per stream.
+// Barge-in clears buffered audio immediately, without waiting for another model response.
+const REPORT_INTERVAL_SAMPLES = 6000;
+const CONTINUOUS_RESERVE_SAMPLES = Math.ceil(sampleRate * 0.2);
+
+class PlaybackProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.queue = [];
+    this.queuedSamples = 0;
+    this.failed = false;
+    this.playedByStream = new Map();
+    this.interruptedStreams = new Set();
+    this.sinceReport = 0;
+    this.ranges = false;
+    this.pendingPlayed = new Map();
+    this.buffering = false;
+    this.bufferingWaited = 0;
+    this.underrunStream = undefined;
+    this.port.onmessage = (event) => {
+      const message = event.data;
+      if (message.type === "configure") {
+        this.ranges = message.ranges === true;
+        // Continuous output arrives at roughly playback speed, so delivery jitter
+        // needs a small reserve. Response audio can still start immediately.
+        this.buffering = this.ranges;
+        this.bufferingWaited = 0;
+      } else if (message.type === "chunk") {
+        if (this.failed || this.interruptedStreams.has(message.streamId) || message.pcm.byteLength === 0)
+          return;
+        if (this.queuedSamples + message.pcm.byteLength / 2 > sampleRate * 30) {
+          for (const streamId of new Set(this.queue.map((entry) => entry.streamId))) this.report(streamId);
+          this.queue = [];
+          this.queuedSamples = 0;
+          this.failed = true;
+          this.port.postMessage({
+            type: "playout.error",
+            message: "Playback is more than 30 seconds behind",
+          });
+          return;
+        }
+        if (!this.ranges && !this.playedByStream.has(message.streamId)) {
+          // Declare known zero playout before the first render, so immediate
+          // barge-in is distinguishable from a legacy client with no receipts.
+          this.playedByStream.set(message.streamId, 0);
+          this.report(message.streamId);
+        }
+        const pcm = new Int16Array(message.pcm);
+        const samples = new Float32Array(pcm.length);
+        for (let i = 0; i < pcm.length; i += 1) samples[i] = pcm[i] / 32768;
+        this.queue.push({ streamId: message.streamId, startSample: message.startSample, samples, offset: 0 });
+        this.queuedSamples += samples.length;
+      } else if (message.type === "interrupt") {
+        this.interrupt(message.streamId);
+      } else if (message.type === "discard") {
+        if (!this.ranges) {
+          // A response receipt describes a contiguous played prefix, so it cannot resume after a gap.
+          for (const streamId of new Set(this.queue.map((entry) => entry.streamId))) this.interrupt(streamId);
+          return;
+        }
+        for (const streamId of this.pendingPlayed.keys()) this.report(streamId);
+        for (const entry of this.queue) {
+          this.port.postMessage({
+            type: "playout.range",
+            streamId: entry.streamId,
+            startSample: entry.startSample + entry.offset,
+            endSample: entry.startSample + entry.samples.length,
+            status: "discarded",
+          });
+        }
+        this.queue = [];
+        this.queuedSamples = 0;
+        this.buffering = true;
+        this.bufferingWaited = 0;
+        this.underrunStream = undefined;
+      } else if (message.type === "reset") {
+        this.queue = [];
+        this.queuedSamples = 0;
+        this.failed = false;
+        this.playedByStream.clear();
+        this.interruptedStreams.clear();
+        this.sinceReport = 0;
+        this.pendingPlayed.clear();
+        this.buffering = this.ranges;
+        this.bufferingWaited = 0;
+        this.underrunStream = undefined;
+      }
+    };
+  }
+
+  interrupt(streamId) {
+    this.interruptedStreams.add(streamId);
+    this.queue = this.queue.filter((entry) => entry.streamId !== streamId);
+    this.queuedSamples = this.queue.reduce((total, entry) => total + entry.samples.length - entry.offset, 0);
+    if (this.ranges && this.queue.length === 0) {
+      this.buffering = true;
+      this.bufferingWaited = 0;
+    }
+    if (this.underrunStream === streamId) this.underrunStream = undefined;
+    this.report(streamId);
+  }
+
+  report(streamId) {
+    if (this.ranges) {
+      const range = this.pendingPlayed.get(streamId);
+      if (range) this.port.postMessage({ type: "playout.range", streamId, ...range, status: "played" });
+      this.pendingPlayed.delete(streamId);
+      return;
+    }
+    const played = this.playedByStream.get(streamId) ?? 0;
+    this.port.postMessage({ type: "playout", streamId, playedThroughSample: played });
+  }
+
+  process(_inputs, outputs) {
+    const output = outputs[0]?.[0];
+    if (!output) return true;
+    if (this.buffering) {
+      const queued = this.queuedSamples;
+      // A finite tail still plays if the source stops before filling the reserve.
+      if (
+        queued === 0 ||
+        (queued < CONTINUOUS_RESERVE_SAMPLES && this.bufferingWaited < CONTINUOUS_RESERVE_SAMPLES)
+      ) {
+        this.bufferingWaited = queued === 0 ? 0 : this.bufferingWaited + output.length;
+        output.fill(0);
+        return true;
+      }
+      this.buffering = false;
+      this.bufferingWaited = 0;
+      if (this.underrunStream !== undefined) {
+        this.port.postMessage({
+          type: "playout.rebuffered",
+          streamId: this.underrunStream,
+          bufferedMs: (queued / sampleRate) * 1000,
+        });
+        this.underrunStream = undefined;
+      }
+    }
+    let written = 0;
+    let streamId;
+    while (written < output.length && this.queue.length > 0) {
+      const head = this.queue[0];
+      streamId = head.streamId;
+      const take = Math.min(output.length - written, head.samples.length - head.offset);
+      if (this.ranges) {
+        const startSample = head.startSample + head.offset;
+        const previous = this.pendingPlayed.get(head.streamId);
+        if (previous && previous.endSample !== startSample) this.report(head.streamId);
+        this.pendingPlayed.set(head.streamId, {
+          startSample: this.pendingPlayed.get(head.streamId)?.startSample ?? startSample,
+          endSample: startSample + take,
+        });
+      }
+      output.set(head.samples.subarray(head.offset, head.offset + take), written);
+      head.offset += take;
+      this.queuedSamples -= take;
+      written += take;
+      this.playedByStream.set(
+        head.streamId,
+        Math.max(this.playedByStream.get(head.streamId) ?? 0, head.startSample + head.offset),
+      );
+      this.sinceReport += take;
+      if (head.offset === head.samples.length) {
+        this.queue.shift();
+        if (this.queue.length === 0 || this.queue[0].streamId !== head.streamId) {
+          this.report(head.streamId);
+          this.sinceReport = 0;
+        }
+      }
+    }
+    if (written < output.length) {
+      output.fill(0, written);
+      if (this.ranges) {
+        this.buffering = true;
+        this.bufferingWaited = 0;
+        this.underrunStream = streamId ?? [...this.playedByStream.keys()].at(-1);
+        this.port.postMessage({ type: "playout.underrun", streamId: this.underrunStream });
+      }
+    }
+    if (this.sinceReport >= REPORT_INTERVAL_SAMPLES && this.queue.length > 0) {
+      this.report(this.queue[0].streamId);
+      this.sinceReport = 0;
+    }
+    return true;
+  }
+}
+
+registerProcessor("xo-playback", PlaybackProcessor);

package/dist/browser.d.ts

@@ -0,0 +1 @@
+export * from "./internal/browser/index.js";

package/dist/browser.js

@@ -0,0 +1 @@
+export * from "./internal/browser/index.js";

package/dist/internal/browser/browser-voice-client.d.ts

@@ -0,0 +1,33 @@
+import type { VoiceStartRequest } from "../harness/index.js";
+export interface BrowserVoiceClientCallbacks {
+    onEvent(event: Record<string, unknown>): void;
+    onStarted(sessionId: string): void;
+    onError(message: string): void;
+    onClosed(reason: string): void;
+    onStats(delta: {
+        sentBytes?: number;
+        receivedBytes?: number;
+    }): void;
+}
+export interface BrowserVoiceClientOptions {
+    socketUrl: string | URL;
+    start: VoiceStartRequest;
+    workletUrls: {
+        capture: string | URL;
+        playback: string | URL;
+    };
+    /** Cancels pending setup only. After start resolves, use stop() for a graceful hang-up. */
+    signal?: AbortSignal;
+}
+/** One live voice conversation: microphone up over the socket, model audio down to the speakers. */
+export declare class BrowserVoiceClient {
+    #private;
+    private constructor();
+    /** Resolves after submitting start locally; onStarted reports the server's readiness frame. */
+    static start(options: BrowserVoiceClientOptions, callbacks: BrowserVoiceClientCallbacks): Promise<BrowserVoiceClient>;
+    stop(): Promise<void>;
+    /** Submits locally after onStarted; throws if the session cannot accept the message. */
+    sendText(text: string): void;
+    setMicrophoneMuted(muted: boolean): void;
+    discardPlayback(): void;
+}