xo-harness 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -5
- package/dist/browser/worklets/capture-processor.js +34 -0
- package/dist/browser/worklets/playback-processor.js +188 -0
- package/dist/browser.d.ts +1 -0
- package/dist/browser.js +1 -0
- package/dist/internal/browser/browser-voice-client.d.ts +33 -0
- package/dist/internal/browser/browser-voice-client.js +316 -0
- package/dist/internal/browser/index.d.ts +2 -0
- package/dist/internal/browser/index.js +1 -0
- package/dist/internal/harness/conversation-context.d.ts +45 -0
- package/dist/internal/harness/conversation-context.js +79 -0
- package/dist/internal/harness/conversation-projection.d.ts +55 -0
- package/dist/internal/harness/conversation-projection.js +145 -0
- package/dist/internal/harness/event-stream.d.ts +23 -2
- package/dist/internal/harness/event-stream.js +148 -18
- package/dist/internal/harness/index.d.ts +7 -1
- package/dist/internal/harness/index.js +7 -1
- package/dist/internal/harness/message.d.ts +24 -11
- package/dist/internal/harness/message.js +271 -77
- package/dist/internal/harness/report-diff.d.ts +3 -0
- package/dist/internal/harness/report-diff.js +10 -0
- package/dist/internal/harness/report.d.ts +17 -14
- package/dist/internal/harness/report.js +41 -24
- package/dist/internal/harness/runtime-limits.d.ts +18 -0
- package/dist/internal/harness/runtime-limits.js +19 -0
- package/dist/internal/harness/session-persistence.d.ts +16 -0
- package/dist/internal/harness/session-persistence.js +106 -0
- package/dist/internal/harness/shadow.d.ts +5 -5
- package/dist/internal/harness/shadow.js +214 -52
- package/dist/internal/harness/socket-bridge.d.ts +45 -4
- package/dist/internal/harness/socket-bridge.js +215 -46
- package/dist/internal/harness/task-supervisor.d.ts +6 -2
- package/dist/internal/harness/task-supervisor.js +97 -15
- package/dist/internal/harness/tool-calls.d.ts +40 -0
- package/dist/internal/harness/tool-calls.js +132 -0
- package/dist/internal/harness/tool-policy.d.ts +2 -2
- package/dist/internal/harness/tool-runtime.d.ts +2 -2
- package/dist/internal/harness/tool-runtime.js +37 -40
- package/dist/internal/harness/tools.d.ts +21 -0
- package/dist/internal/harness/tools.js +7 -1
- package/dist/internal/harness/usage-tracker.d.ts +39 -0
- package/dist/internal/harness/usage-tracker.js +104 -0
- package/dist/internal/harness/voice-session.d.ts +39 -8
- package/dist/internal/harness/voice-session.js +353 -58
- package/dist/internal/harness/xo.d.ts +7 -12
- package/dist/internal/harness/xo.js +25 -13
- package/dist/internal/protocol/async-queue.d.ts +22 -1
- package/dist/internal/protocol/async-queue.js +86 -12
- package/dist/internal/protocol/audio.d.ts +16 -2
- package/dist/internal/protocol/audio.js +23 -5
- package/dist/internal/protocol/backend-output.d.ts +70 -0
- package/dist/internal/protocol/backend-output.js +39 -0
- package/dist/internal/protocol/event-json.d.ts +3 -0
- package/dist/internal/protocol/event-json.js +15 -0
- package/dist/internal/protocol/events.d.ts +342 -4
- package/dist/internal/protocol/events.js +53 -25
- package/dist/internal/protocol/index.d.ts +5 -0
- package/dist/internal/protocol/index.js +5 -0
- package/dist/internal/protocol/output-source.d.ts +18 -0
- package/dist/internal/protocol/output-source.js +15 -0
- package/dist/internal/protocol/paced-audio.d.ts +27 -0
- package/dist/internal/protocol/paced-audio.js +117 -0
- package/dist/internal/protocol/parts.d.ts +154 -0
- package/dist/internal/protocol/parts.js +32 -6
- package/dist/internal/protocol/provider.d.ts +308 -1
- package/dist/internal/protocol/provider.js +102 -3
- package/dist/internal/protocol/records.d.ts +773 -0
- package/dist/internal/protocol/records.js +46 -0
- package/dist/internal/protocol/tools.d.ts +40 -0
- package/dist/internal/protocol/tools.js +12 -0
- package/dist/internal/protocol/transcript.d.ts +19 -0
- package/dist/internal/protocol/transcript.js +53 -0
- package/dist/internal/provider/contract.d.ts +25 -2
- package/dist/internal/provider/event-queue.d.ts +17 -0
- package/dist/internal/provider/event-queue.js +78 -0
- package/dist/internal/provider/grok-voice.d.ts +12 -9
- package/dist/internal/provider/grok-voice.js +26 -15
- package/dist/internal/provider/index.d.ts +1 -0
- package/dist/internal/provider/index.js +1 -0
- package/dist/internal/provider/live-session.d.ts +29 -0
- package/dist/internal/provider/live-session.js +840 -0
- package/dist/internal/provider/node-socket.js +6 -0
- package/dist/internal/provider/openai-live.d.ts +62 -0
- package/dist/internal/provider/openai-live.js +160 -0
- package/dist/internal/provider/openai-realtime.d.ts +9 -5
- package/dist/internal/provider/openai-realtime.js +24 -10
- package/dist/internal/provider/realtime-session.d.ts +12 -1
- package/dist/internal/provider/realtime-session.js +373 -70
- package/dist/internal/provider/realtime-socket.d.ts +3 -1
- package/dist/internal/provider/tool-status-context.d.ts +10 -0
- package/dist/internal/provider/tool-status-context.js +30 -0
- package/dist/internal/provider/workers-socket.js +63 -15
- package/dist/internal/provider/workers.d.ts +1 -0
- package/dist/internal/provider/workers.js +1 -0
- package/dist/internal/provider-fake/replay-voice-provider.d.ts +9 -11
- package/dist/internal/provider-fake/replay-voice-provider.js +47 -23
- package/dist/internal/storage/event-store.d.ts +22 -4
- package/dist/internal/storage/jsonl-event-store.d.ts +4 -4
- package/dist/internal/storage/jsonl-event-store.js +21 -16
- package/dist/internal/storage/memory-event-store.d.ts +4 -3
- package/dist/internal/storage/memory-event-store.js +8 -2
- package/dist/internal/storage/memory.d.ts +1 -1
- package/dist/internal/testkit/events.d.ts +14 -0
- package/dist/internal/testkit/events.js +33 -0
- package/dist/internal/testkit/runtime.d.ts +3 -0
- package/dist/internal/testkit/runtime.js +3 -0
- package/dist/internal/testkit/trajectory.d.ts +23 -0
- package/dist/internal/testkit/trajectory.js +58 -0
- package/dist/internal/tools-openai/index.d.ts +2 -0
- package/dist/internal/tools-openai/index.js +79 -72
- package/dist/internal/tools-openai/responses.d.ts +5 -1
- package/dist/internal/tools-openai/responses.js +53 -5
- package/dist/testing.d.ts +1 -0
- package/dist/testing.js +1 -0
- package/package.json +7 -1
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/** Opt-in per-session limits. Omitted fields preserve full replay and unlimited storage waits. */
|
|
2
|
+
export interface SessionRuntimeLimits {
|
|
3
|
+
/** Oldest events are evicted together; lagging readers receive EventReplayUnavailableError. */
|
|
4
|
+
maxReplayEvents?: number;
|
|
5
|
+
/** PCM bytes retained in the shared replay window, independent of the EventStore's retention. */
|
|
6
|
+
maxReplayAudioBytes?: number;
|
|
7
|
+
/** Maximum underlying appends in flight. Overflow fails the session instead of dropping records. */
|
|
8
|
+
maxPendingAppends?: number;
|
|
9
|
+
/** PCM bytes held by underlying appends, independent of chunk size. */
|
|
10
|
+
maxPendingAudioBytes?: number;
|
|
11
|
+
/** Deadline for each store operation. A timeout does not cancel an underlying external write. */
|
|
12
|
+
persistenceTimeoutMs?: number;
|
|
13
|
+
}
|
|
14
|
+
export declare class SessionPersistenceError extends Error {
|
|
15
|
+
readonly code: "persistence_failed" | "persistence_timeout" | "pending_appends_exceeded" | "pending_audio_bytes_exceeded";
|
|
16
|
+
constructor(code: "persistence_failed" | "persistence_timeout" | "pending_appends_exceeded" | "pending_audio_bytes_exceeded", message: string, options?: ErrorOptions);
|
|
17
|
+
}
|
|
18
|
+
export declare function validateRuntimeLimits(limits?: SessionRuntimeLimits): Readonly<SessionRuntimeLimits>;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export class SessionPersistenceError extends Error {
|
|
2
|
+
code;
|
|
3
|
+
constructor(code, message, options) {
|
|
4
|
+
super(message, options);
|
|
5
|
+
this.code = code;
|
|
6
|
+
this.name = "SessionPersistenceError";
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
export function validateRuntimeLimits(limits = {}) {
|
|
10
|
+
for (const [key, value] of Object.entries(limits)) {
|
|
11
|
+
if (value === undefined)
|
|
12
|
+
continue;
|
|
13
|
+
const minimum = key.endsWith("Bytes") || key === "maxReplayEvents" ? 0 : 1;
|
|
14
|
+
const maximum = key === "persistenceTimeoutMs" ? 2_147_483_647 : Number.MAX_SAFE_INTEGER;
|
|
15
|
+
if (!Number.isSafeInteger(value) || value < minimum || value > maximum)
|
|
16
|
+
throw new Error(`${key} must be an integer between ${minimum} and ${maximum}`);
|
|
17
|
+
}
|
|
18
|
+
return Object.freeze({ ...limits });
|
|
19
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { HarnessEvent, HarnessRecord, UnsequencedHarnessEvent } from "../protocol/index.js";
|
|
2
|
+
import type { EventStore } from "../storage/index.js";
|
|
3
|
+
import { SessionPersistenceError, type SessionRuntimeLimits } from "./runtime-limits.js";
|
|
4
|
+
/** Session-local admission/deadlines around the host store; never bypasses durable publication. */
|
|
5
|
+
export declare class SessionPersistence implements EventStore {
|
|
6
|
+
#private;
|
|
7
|
+
onFailure: ((error: SessionPersistenceError) => void) | undefined;
|
|
8
|
+
constructor(store: EventStore, limits: Readonly<SessionRuntimeLimits>);
|
|
9
|
+
stats(): {
|
|
10
|
+
pendingAppends: number;
|
|
11
|
+
pendingAppendAudioBytes: number;
|
|
12
|
+
};
|
|
13
|
+
append(event: UnsequencedHarnessEvent): Promise<HarnessEvent>;
|
|
14
|
+
flush(sessionId: string): Promise<void>;
|
|
15
|
+
list(sessionId: string): Promise<readonly HarnessRecord[]>;
|
|
16
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { SessionPersistenceError } from "./runtime-limits.js";
|
|
2
|
+
/** Session-local admission/deadlines around the host store; never bypasses durable publication. */
|
|
3
|
+
export class SessionPersistence {
|
|
4
|
+
#store;
|
|
5
|
+
#limits;
|
|
6
|
+
#pendingAppends = 0;
|
|
7
|
+
#pendingAudioBytes = 0;
|
|
8
|
+
#failure;
|
|
9
|
+
#failureWaiters = new Set();
|
|
10
|
+
onFailure;
|
|
11
|
+
constructor(store, limits) {
|
|
12
|
+
this.#store = store;
|
|
13
|
+
this.#limits = limits;
|
|
14
|
+
}
|
|
15
|
+
stats() {
|
|
16
|
+
return { pendingAppends: this.#pendingAppends, pendingAppendAudioBytes: this.#pendingAudioBytes };
|
|
17
|
+
}
|
|
18
|
+
append(event) {
|
|
19
|
+
if (this.#failure)
|
|
20
|
+
return Promise.reject(this.#failure);
|
|
21
|
+
if (this.#pendingAppends >= (this.#limits.maxPendingAppends ?? Infinity))
|
|
22
|
+
return Promise.reject(this.#fail(new SessionPersistenceError("pending_appends_exceeded", `Pending event appends exceeded ${this.#limits.maxPendingAppends}; session recording is incomplete`)));
|
|
23
|
+
const bytes = event.type === "audio.input" || event.type === "audio.output" ? event.chunk.data.byteLength : 0;
|
|
24
|
+
if (this.#pendingAudioBytes + bytes > (this.#limits.maxPendingAudioBytes ?? Infinity))
|
|
25
|
+
return Promise.reject(this.#fail(new SessionPersistenceError("pending_audio_bytes_exceeded", `Pending audio appends exceeded ${this.#limits.maxPendingAudioBytes} bytes; session recording is incomplete`)));
|
|
26
|
+
this.#pendingAppends++;
|
|
27
|
+
this.#pendingAudioBytes += bytes;
|
|
28
|
+
// Invoke synchronously: the store reserves ordering before any provider forwarding.
|
|
29
|
+
let append;
|
|
30
|
+
try {
|
|
31
|
+
append = this.#store.append(event);
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
append = Promise.reject(this.#operationFailed(error, "append"));
|
|
35
|
+
}
|
|
36
|
+
const settled = append.finally(() => {
|
|
37
|
+
// Counts underlying writes, including writes still outstanding after a deadline.
|
|
38
|
+
this.#pendingAppends--;
|
|
39
|
+
this.#pendingAudioBytes -= bytes;
|
|
40
|
+
});
|
|
41
|
+
return this.#observe(settled, "append");
|
|
42
|
+
}
|
|
43
|
+
flush(sessionId) {
|
|
44
|
+
return this.#call(() => this.#store.flush(sessionId), "flush");
|
|
45
|
+
}
|
|
46
|
+
list(sessionId) {
|
|
47
|
+
return this.#call(() => this.#store.list(sessionId), "list");
|
|
48
|
+
}
|
|
49
|
+
#call(operation, label) {
|
|
50
|
+
if (this.#failure)
|
|
51
|
+
return Promise.reject(this.#failure);
|
|
52
|
+
try {
|
|
53
|
+
return this.#observe(operation(), label);
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
return Promise.reject(this.#operationFailed(error, label));
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
async #observe(operation, label) {
|
|
60
|
+
let timer;
|
|
61
|
+
let rejectFailure;
|
|
62
|
+
try {
|
|
63
|
+
const timeout = this.#limits.persistenceTimeoutMs;
|
|
64
|
+
const pending = [operation];
|
|
65
|
+
pending.push(new Promise((_resolve, reject) => {
|
|
66
|
+
if (this.#failure)
|
|
67
|
+
reject(this.#failure);
|
|
68
|
+
else {
|
|
69
|
+
rejectFailure = reject;
|
|
70
|
+
this.#failureWaiters.add(reject);
|
|
71
|
+
}
|
|
72
|
+
}));
|
|
73
|
+
if (timeout !== undefined)
|
|
74
|
+
pending.push(new Promise((_resolve, reject) => {
|
|
75
|
+
timer = setTimeout(() => reject(this.#fail(new SessionPersistenceError("persistence_timeout", `EventStore ${label} exceeded ${timeout} ms; persistence remains unconfirmed`))), timeout);
|
|
76
|
+
}));
|
|
77
|
+
const value = await Promise.race(pending);
|
|
78
|
+
if (this.#failure)
|
|
79
|
+
throw this.#failure;
|
|
80
|
+
return value;
|
|
81
|
+
}
|
|
82
|
+
catch (error) {
|
|
83
|
+
throw this.#operationFailed(error, label);
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
clearTimeout(timer);
|
|
87
|
+
if (rejectFailure)
|
|
88
|
+
this.#failureWaiters.delete(rejectFailure);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
#operationFailed(error, label) {
|
|
92
|
+
return this.#fail(error instanceof SessionPersistenceError
|
|
93
|
+
? error
|
|
94
|
+
: new SessionPersistenceError("persistence_failed", `EventStore ${label} failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error }));
|
|
95
|
+
}
|
|
96
|
+
#fail(error) {
|
|
97
|
+
if (!this.#failure) {
|
|
98
|
+
this.#failure = error;
|
|
99
|
+
for (const reject of this.#failureWaiters)
|
|
100
|
+
reject(error);
|
|
101
|
+
this.#failureWaiters.clear();
|
|
102
|
+
this.onFailure?.(error);
|
|
103
|
+
}
|
|
104
|
+
return this.#failure;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type HarnessRecord } from "../protocol/index.js";
|
|
2
2
|
import type { VoiceProvider } from "../provider/index.js";
|
|
3
3
|
import { type EventStore } from "../storage/memory.js";
|
|
4
4
|
import { type SessionReport } from "./report.js";
|
|
@@ -6,7 +6,7 @@ import type { ToolPolicy } from "./tool-policy.js";
|
|
|
6
6
|
import type { VoiceTool } from "./tools.js";
|
|
7
7
|
export interface ShadowRunOptions {
|
|
8
8
|
/** The recorded session whose microphone audio is re-driven. */
|
|
9
|
-
events: readonly
|
|
9
|
+
events: readonly HarnessRecord[];
|
|
10
10
|
/** The provider to shadow-test with the recorded input. */
|
|
11
11
|
provider: VoiceProvider;
|
|
12
12
|
/** Tools to execute during the shadow run; use substitutes when side effects are unwanted. */
|
|
@@ -20,7 +20,7 @@ export interface ShadowRunOptions {
|
|
|
20
20
|
sessionId?: string;
|
|
21
21
|
/** Defaults to the recorded session's instructions. */
|
|
22
22
|
instructions?: string;
|
|
23
|
-
/**
|
|
23
|
+
/** Waits this long after meaningful output, with no pending backend, tool, or task work. */
|
|
24
24
|
quietMs?: number;
|
|
25
25
|
timeoutMs?: number;
|
|
26
26
|
}
|
|
@@ -29,7 +29,7 @@ export interface ShadowRunOptions {
|
|
|
29
29
|
* provider and returns the resulting session report for comparison with the original.
|
|
30
30
|
* The shadow run executes supplied tools and applies only the policy provided here;
|
|
31
31
|
* a recording's denials do not establish permissions for a new session.
|
|
32
|
-
*
|
|
33
|
-
*
|
|
32
|
+
* Continuous-input providers receive paced PCM and silence through settlement.
|
|
33
|
+
* Other providers retain the existing push-input behavior.
|
|
34
34
|
*/
|
|
35
35
|
export declare function runShadowSession(options: ShadowRunOptions): Promise<SessionReport>;
|
|
@@ -1,22 +1,35 @@
|
|
|
1
|
+
import { PacedAudioInput, requireRecordedAudioHistory, } from "../protocol/index.js";
|
|
1
2
|
import { MemoryEventStore } from "../storage/memory.js";
|
|
2
3
|
import { summarizeSession } from "./report.js";
|
|
4
|
+
import { ToolCallTracker } from "./tool-calls.js";
|
|
3
5
|
import { XO } from "./xo.js";
|
|
4
6
|
/**
|
|
5
7
|
* The silent-test primitive: re-drives a recorded session's input audio against another
|
|
6
8
|
* provider and returns the resulting session report for comparison with the original.
|
|
7
9
|
* The shadow run executes supplied tools and applies only the policy provided here;
|
|
8
10
|
* a recording's denials do not establish permissions for a new session.
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
+
* Continuous-input providers receive paced PCM and silence through settlement.
|
|
12
|
+
* Other providers retain the existing push-input behavior.
|
|
11
13
|
*/
|
|
12
14
|
export async function runShadowSession(options) {
|
|
13
15
|
const recordedStart = options.events.find((event) => event.type === "session.started");
|
|
14
|
-
|
|
15
|
-
|
|
16
|
+
// Validate every required input before opening a provider or executing tools.
|
|
17
|
+
const inputs = requireRecordedAudioHistory(options.events.filter((event) => event.type === "audio.input")).filter((event) => event.type === "audio.input");
|
|
18
|
+
const firstInput = inputs[0];
|
|
19
|
+
if (!firstInput)
|
|
16
20
|
throw new Error("Recorded session has no input audio to re-drive");
|
|
17
21
|
const instructions = options.instructions ?? recordedStart?.instructions;
|
|
18
22
|
const quietMs = options.quietMs ?? 3_000;
|
|
19
23
|
const timeoutMs = options.timeoutMs ?? 60_000;
|
|
24
|
+
if (!Number.isFinite(quietMs) || quietMs < 0 || !Number.isFinite(timeoutMs) || timeoutMs < 1) {
|
|
25
|
+
throw new Error("Shadow quietMs must be nonnegative and timeoutMs must be positive and finite");
|
|
26
|
+
}
|
|
27
|
+
const continuousInput = options.provider.capabilities.inputCadence === "continuous";
|
|
28
|
+
const continuousOutput = options.provider.capabilities.outputLifecycle === "continuous";
|
|
29
|
+
if (continuousInput &&
|
|
30
|
+
inputs.some((input) => input.chunk.channels !== 1 || input.chunk.sampleRate !== firstInput.chunk.sampleRate)) {
|
|
31
|
+
throw new Error("Continuous shadow input requires one consistent mono PCM sample rate");
|
|
32
|
+
}
|
|
20
33
|
const harness = new XO({
|
|
21
34
|
store: options.store ?? new MemoryEventStore(),
|
|
22
35
|
...(options.tools === undefined ? {} : { tools: options.tools }),
|
|
@@ -29,18 +42,130 @@ export async function runShadowSession(options) {
|
|
|
29
42
|
...(options.sessionId === undefined ? {} : { sessionId: options.sessionId }),
|
|
30
43
|
...(instructions === undefined ? {} : { instructions }),
|
|
31
44
|
});
|
|
45
|
+
const deadline = performance.now() + timeoutMs;
|
|
46
|
+
const inputController = new AbortController();
|
|
47
|
+
let pacedInput;
|
|
48
|
+
let closeReason = "shadow_complete";
|
|
49
|
+
const inputDeadline = setTimeout(() => {
|
|
50
|
+
closeReason = "shadow_timeout";
|
|
51
|
+
inputController.abort(new Error("Shadow input exceeded the run deadline"));
|
|
52
|
+
}, timeoutMs);
|
|
32
53
|
try {
|
|
33
|
-
await
|
|
34
|
-
|
|
35
|
-
|
|
54
|
+
await waitOrAbort(session.waitUntilReady(), inputController.signal);
|
|
55
|
+
if (continuousInput) {
|
|
56
|
+
const sampleRate = firstInput.chunk.sampleRate;
|
|
57
|
+
let sequence = 0;
|
|
58
|
+
let startSample = 0;
|
|
59
|
+
let frameSubmitted = Promise.withResolvers();
|
|
60
|
+
pacedInput = new PacedAudioInput({
|
|
61
|
+
sampleRate,
|
|
62
|
+
signal: inputController.signal,
|
|
63
|
+
send: async (data, syntheticSilence) => {
|
|
64
|
+
const chunk = {
|
|
65
|
+
streamId: "shadow-input",
|
|
66
|
+
sequence: sequence++,
|
|
67
|
+
encoding: "pcm_s16le",
|
|
68
|
+
sampleRate,
|
|
69
|
+
channels: 1,
|
|
70
|
+
startSample,
|
|
71
|
+
data: Uint8Array.from(data),
|
|
72
|
+
};
|
|
73
|
+
await session.sendAudio(chunk, { syntheticSilence });
|
|
74
|
+
startSample += data.byteLength / 2;
|
|
75
|
+
const submitted = frameSubmitted;
|
|
76
|
+
frameSubmitted = Promise.withResolvers();
|
|
77
|
+
submitted.resolve();
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
// A rejected input clock must not become an unhandled rejection while the
|
|
81
|
+
// recording is waiting for its next source timestamp.
|
|
82
|
+
void pacedInput.done.catch((error) => inputController.abort(error));
|
|
83
|
+
await driveContinuousInput(inputs, pacedInput, deadline, recordedStart?.capabilities.inputCadence !== "continuous", async (targetSample) => {
|
|
84
|
+
while (startSample < targetSample)
|
|
85
|
+
await waitOrAbort(frameSubmitted.promise, inputController.signal);
|
|
86
|
+
});
|
|
36
87
|
}
|
|
37
|
-
|
|
88
|
+
else {
|
|
89
|
+
for (const input of inputs)
|
|
90
|
+
await session.sendAudio(input.chunk);
|
|
91
|
+
}
|
|
92
|
+
const settlement = settleAfterOutput(session, quietMs, deadline, continuousOutput);
|
|
93
|
+
const result = await (pacedInput
|
|
94
|
+
? Promise.race([settlement, pacedInput.done.then(() => "closed")])
|
|
95
|
+
: settlement);
|
|
96
|
+
if (result === "timeout")
|
|
97
|
+
closeReason = "shadow_timeout";
|
|
38
98
|
}
|
|
39
99
|
finally {
|
|
40
|
-
|
|
100
|
+
clearTimeout(inputDeadline);
|
|
101
|
+
inputController.abort("shadow_complete");
|
|
102
|
+
await pacedInput?.done.catch(() => undefined);
|
|
103
|
+
await session.close(closeReason);
|
|
41
104
|
}
|
|
42
105
|
return summarizeSession(await session.history());
|
|
43
106
|
}
|
|
107
|
+
async function driveContinuousInput(inputs, pacedInput, deadline, preserveCapturePauses, waitUntilSample) {
|
|
108
|
+
const first = inputs[0];
|
|
109
|
+
if (!first)
|
|
110
|
+
return;
|
|
111
|
+
const sourceStart = first.sessionTimeMs - (first.chunk.data.byteLength / 2 / first.chunk.sampleRate) * 1000;
|
|
112
|
+
const frameSamples = Math.round(first.chunk.sampleRate / 10);
|
|
113
|
+
const streams = new Map();
|
|
114
|
+
let frame = new Uint8Array(frameSamples * 2);
|
|
115
|
+
let filled = 0;
|
|
116
|
+
let position = 0;
|
|
117
|
+
let hasSourceAudio = false;
|
|
118
|
+
const flushFrame = async () => {
|
|
119
|
+
if (performance.now() >= deadline)
|
|
120
|
+
throw new Error("Shadow input exceeds the run deadline");
|
|
121
|
+
if (hasSourceAudio)
|
|
122
|
+
await pacedInput.write(frame);
|
|
123
|
+
else
|
|
124
|
+
await waitUntilSample(position);
|
|
125
|
+
frame = new Uint8Array(frameSamples * 2);
|
|
126
|
+
filled = 0;
|
|
127
|
+
hasSourceAudio = false;
|
|
128
|
+
};
|
|
129
|
+
const appendSamples = async (count, data) => {
|
|
130
|
+
let consumed = 0;
|
|
131
|
+
while (consumed < count) {
|
|
132
|
+
const take = Math.min(frameSamples - filled, count - consumed);
|
|
133
|
+
if (data) {
|
|
134
|
+
frame.set(data.subarray(consumed * 2, (consumed + take) * 2), filled * 2);
|
|
135
|
+
hasSourceAudio = true;
|
|
136
|
+
}
|
|
137
|
+
filled += take;
|
|
138
|
+
consumed += take;
|
|
139
|
+
position += take;
|
|
140
|
+
if (filled === frameSamples)
|
|
141
|
+
await flushFrame();
|
|
142
|
+
}
|
|
143
|
+
};
|
|
144
|
+
for (const input of inputs) {
|
|
145
|
+
const { chunk } = input;
|
|
146
|
+
const observedOffsetMs = input.sessionTimeMs - (chunk.data.byteLength / 2 / chunk.sampleRate) * 1000 - sourceStart;
|
|
147
|
+
let origin = streams.get(chunk.streamId);
|
|
148
|
+
if (!origin) {
|
|
149
|
+
origin = { sample: chunk.startSample, offsetMs: observedOffsetMs };
|
|
150
|
+
streams.set(chunk.streamId, origin);
|
|
151
|
+
}
|
|
152
|
+
let offsetMs = Math.max(0, origin.offsetMs + ((chunk.startSample - origin.sample) / chunk.sampleRate) * 1_000);
|
|
153
|
+
if (preserveCapturePauses && observedOffsetMs - offsetMs > 250) {
|
|
154
|
+
// On-demand capture can stop while its sample counter stays contiguous. Preserve
|
|
155
|
+
// that pause; continuous recordings keep their sample clock despite receipt jitter.
|
|
156
|
+
offsetMs = observedOffsetMs;
|
|
157
|
+
streams.set(chunk.streamId, { sample: chunk.startSample, offsetMs });
|
|
158
|
+
}
|
|
159
|
+
// Coalesce source packets before awaiting the clock: a 20 ms packet must not
|
|
160
|
+
// acquire 80 ms of padding merely because the transport uses 100 ms frames.
|
|
161
|
+
// Sample positions preserve gaps, including silence inside a transport frame.
|
|
162
|
+
const targetSample = Math.round((offsetMs * chunk.sampleRate) / 1_000);
|
|
163
|
+
await appendSamples(Math.max(0, targetSample - position));
|
|
164
|
+
await appendSamples(chunk.data.byteLength / 2, input.syntheticSilence ? undefined : chunk.data);
|
|
165
|
+
}
|
|
166
|
+
if (filled)
|
|
167
|
+
await flushFrame();
|
|
168
|
+
}
|
|
44
169
|
function createEventCursor(session) {
|
|
45
170
|
const iterator = session.events()[Symbol.asyncIterator]();
|
|
46
171
|
// A raced-and-lost next() must be reused, not abandoned: an abandoned call would
|
|
@@ -49,7 +174,13 @@ function createEventCursor(session) {
|
|
|
49
174
|
return {
|
|
50
175
|
async next(timeoutMs) {
|
|
51
176
|
pendingNext ??= iterator.next();
|
|
52
|
-
|
|
177
|
+
let timer;
|
|
178
|
+
const result = await Promise.race([
|
|
179
|
+
pendingNext,
|
|
180
|
+
new Promise((resolve) => {
|
|
181
|
+
timer = setTimeout(() => resolve("timeout"), timeoutMs);
|
|
182
|
+
}),
|
|
183
|
+
]).finally(() => clearTimeout(timer));
|
|
53
184
|
if (result === "timeout")
|
|
54
185
|
return "timeout";
|
|
55
186
|
pendingNext = undefined;
|
|
@@ -62,54 +193,72 @@ function createEventCursor(session) {
|
|
|
62
193
|
},
|
|
63
194
|
};
|
|
64
195
|
}
|
|
65
|
-
async function
|
|
196
|
+
async function settleAfterOutput(session, quietMs, deadline, continuousOutput) {
|
|
66
197
|
const cursor = createEventCursor(session);
|
|
67
|
-
|
|
198
|
+
let lastActivity = performance.now();
|
|
199
|
+
let sawOutputSinceWork = false;
|
|
200
|
+
let responseIdle = true;
|
|
201
|
+
const toolCalls = new ToolCallTracker();
|
|
202
|
+
const backends = new Set();
|
|
203
|
+
const hostedTools = new Set();
|
|
68
204
|
try {
|
|
69
205
|
while (true) {
|
|
70
|
-
const
|
|
71
|
-
if (remaining <= 0)
|
|
72
|
-
throw new Error("Shadow provider did not become ready in time");
|
|
73
|
-
const event = await cursor.next(remaining);
|
|
74
|
-
if (event === "timeout")
|
|
75
|
-
throw new Error("Shadow provider did not become ready in time");
|
|
76
|
-
if (event === "done")
|
|
77
|
-
throw new Error("Shadow session ended before the provider was ready");
|
|
78
|
-
if (event.type === "provider.ready")
|
|
79
|
-
return;
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
finally {
|
|
83
|
-
cursor.dispose();
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
async function settleAfterOutput(session, quietMs, timeoutMs) {
|
|
87
|
-
const cursor = createEventCursor(session);
|
|
88
|
-
const deadline = Date.now() + timeoutMs;
|
|
89
|
-
let lastActivity = Date.now();
|
|
90
|
-
let sawOutput = false;
|
|
91
|
-
try {
|
|
92
|
-
while (true) {
|
|
93
|
-
const untilDeadline = deadline - Date.now();
|
|
206
|
+
const untilDeadline = deadline - performance.now();
|
|
94
207
|
if (untilDeadline <= 0)
|
|
95
|
-
return;
|
|
96
|
-
const untilQuiet = quietMs - (
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
208
|
+
return "timeout";
|
|
209
|
+
const untilQuiet = quietMs - (performance.now() - lastActivity);
|
|
210
|
+
const canSettle = sawOutputSinceWork &&
|
|
211
|
+
responseIdle &&
|
|
212
|
+
!toolCalls.hasPending() &&
|
|
213
|
+
backends.size + hostedTools.size === 0;
|
|
214
|
+
const wait = Math.max(0, canSettle ? Math.min(untilQuiet, untilDeadline) : untilDeadline);
|
|
100
215
|
const event = await cursor.next(wait);
|
|
101
|
-
if (event === "timeout")
|
|
216
|
+
if (event === "timeout") {
|
|
217
|
+
if (canSettle && performance.now() - lastActivity >= quietMs)
|
|
218
|
+
return "quiet";
|
|
102
219
|
continue;
|
|
220
|
+
}
|
|
103
221
|
if (event === "done")
|
|
104
|
-
return;
|
|
222
|
+
return "closed";
|
|
105
223
|
const type = event.type;
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
224
|
+
const previousToolCall = "callId" in event && event.callId !== undefined ? toolCalls.get(event.callId) : undefined;
|
|
225
|
+
const toolCall = toolCalls.apply(event);
|
|
226
|
+
// Only a new execution result requires another answer. Retransmissions and
|
|
227
|
+
// delivery diagnostics must not erase an answer already observed.
|
|
228
|
+
let workSettled = toolCall?.status === "result_ready" && toolCall.state !== previousToolCall?.state;
|
|
229
|
+
if (type === "backend.response") {
|
|
230
|
+
// Voice may announce a result before the backend emits its trailing terminal.
|
|
231
|
+
// That terminal releases the pending gate; it does not require another answer.
|
|
232
|
+
if (event.phase === "completed")
|
|
233
|
+
backends.delete(event.responseId);
|
|
234
|
+
else
|
|
235
|
+
backends.add(event.responseId);
|
|
236
|
+
}
|
|
237
|
+
if (type === "backend.tool") {
|
|
238
|
+
const key = JSON.stringify([event.responseId, event.toolId]);
|
|
239
|
+
if (event.phase === "started")
|
|
240
|
+
hostedTools.add(key);
|
|
241
|
+
else
|
|
242
|
+
workSettled = hostedTools.delete(key);
|
|
243
|
+
}
|
|
244
|
+
if (type === "delegation.created" && event.target === "responses" && event.responseId)
|
|
245
|
+
backends.add(event.responseId);
|
|
246
|
+
if (type === "response.state")
|
|
247
|
+
responseIdle = event.state === "idle";
|
|
248
|
+
if (workSettled)
|
|
249
|
+
sawOutputSinceWork = false;
|
|
250
|
+
const spokenAudio = type === "audio.output" && (!continuousOutput || hasAudibleSamples(event.chunk.data));
|
|
251
|
+
const assistantText = (type === "transcript" || type === "transcript.delta" || type === "transcript.fragment") &&
|
|
252
|
+
event.role === "assistant";
|
|
253
|
+
if (spokenAudio || assistantText)
|
|
254
|
+
sawOutputSinceWork = true;
|
|
255
|
+
if (spokenAudio ||
|
|
256
|
+
assistantText ||
|
|
257
|
+
type === "response.state" ||
|
|
258
|
+
type.startsWith("tool.") ||
|
|
259
|
+
type.startsWith("task.") ||
|
|
260
|
+
type.startsWith("backend.")) {
|
|
261
|
+
lastActivity = performance.now();
|
|
113
262
|
}
|
|
114
263
|
}
|
|
115
264
|
}
|
|
@@ -117,6 +266,19 @@ async function settleAfterOutput(session, quietMs, timeoutMs) {
|
|
|
117
266
|
cursor.dispose();
|
|
118
267
|
}
|
|
119
268
|
}
|
|
120
|
-
function
|
|
121
|
-
|
|
269
|
+
function hasAudibleSamples(data) {
|
|
270
|
+
const samples = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
271
|
+
let squares = 0;
|
|
272
|
+
for (let offset = 0; offset < data.byteLength; offset += 2)
|
|
273
|
+
squares += (samples.getInt16(offset, true) / 32_768) ** 2;
|
|
274
|
+
return Math.sqrt(squares / Math.max(1, data.byteLength / 2)) >= 0.003;
|
|
275
|
+
}
|
|
276
|
+
function waitOrAbort(promise, signal) {
|
|
277
|
+
return new Promise((resolve, reject) => {
|
|
278
|
+
const abort = () => reject(signal.reason);
|
|
279
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
280
|
+
if (signal.aborted)
|
|
281
|
+
abort();
|
|
282
|
+
void promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
|
|
283
|
+
});
|
|
122
284
|
}
|
|
@@ -4,7 +4,7 @@ import type { VoiceSession } from "./voice-session.js";
|
|
|
4
4
|
/**
|
|
5
5
|
* The client wire protocol every XO transport speaks:
|
|
6
6
|
* - text frames are JSON control messages (start / playout / stop up; started / event /
|
|
7
|
-
* error / closed down),
|
|
7
|
+
* interrupt / error / closed down),
|
|
8
8
|
* - binary frames are PCM16 mono audio (microphone up; length-prefixed model audio down:
|
|
9
9
|
* u32le header length, JSON header, PCM bytes).
|
|
10
10
|
*
|
|
@@ -30,6 +30,15 @@ export declare const VoiceClientMessageSchema: z.ZodDiscriminatedUnion<[z.ZodObj
|
|
|
30
30
|
type: z.ZodLiteral<"playout">;
|
|
31
31
|
streamId: z.ZodString;
|
|
32
32
|
playedThroughSample: z.ZodNumber;
|
|
33
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
34
|
+
streamId: z.ZodString;
|
|
35
|
+
startSample: z.ZodInt;
|
|
36
|
+
endSample: z.ZodInt;
|
|
37
|
+
status: z.ZodEnum<{
|
|
38
|
+
discarded: "discarded";
|
|
39
|
+
played: "played";
|
|
40
|
+
}>;
|
|
41
|
+
type: z.ZodLiteral<"playout.range">;
|
|
33
42
|
}, z.core.$strip>, z.ZodObject<{
|
|
34
43
|
type: z.ZodLiteral<"stop">;
|
|
35
44
|
reason: z.ZodOptional<z.ZodString>;
|
|
@@ -40,13 +49,41 @@ export type VoiceStartRequest = Omit<Extract<VoiceClientMessage, {
|
|
|
40
49
|
type: "start";
|
|
41
50
|
}>, "type">;
|
|
42
51
|
export interface VoiceBridgeSocket {
|
|
52
|
+
/** Throw if the frame cannot be accepted locally; returning does not confirm client receipt. */
|
|
43
53
|
sendText(text: string): void;
|
|
44
54
|
sendBinary(frame: ArrayBuffer): void;
|
|
45
55
|
}
|
|
56
|
+
export interface VoiceBridgeStartupContext {
|
|
57
|
+
/** Aborted on disconnect only until the factory returns; safe to pass to XO.startSession. */
|
|
58
|
+
startupSignal: AbortSignal;
|
|
59
|
+
}
|
|
60
|
+
export interface VoiceBridgeStartedContext {
|
|
61
|
+
/** Cancels opening work on close, independently of the provider's owner signal. */
|
|
62
|
+
signal: AbortSignal;
|
|
63
|
+
}
|
|
64
|
+
export interface VoiceBridgeCloseResult {
|
|
65
|
+
/** Whether pending creation/late cleanup settled before the bridge's deadline, not provider finalization. */
|
|
66
|
+
readonly startup: "settled" | "unconfirmed";
|
|
67
|
+
}
|
|
46
68
|
export interface VoiceSocketBridgeOptions {
|
|
47
69
|
socket: VoiceBridgeSocket;
|
|
48
70
|
/** Owns provider resolution and XO construction; the bridge owns only the wire. */
|
|
49
|
-
startSession(start: VoiceStartRequest): Promise<VoiceSession>;
|
|
71
|
+
startSession(start: VoiceStartRequest, context: VoiceBridgeStartupContext): Promise<VoiceSession>;
|
|
72
|
+
/**
|
|
73
|
+
* Runs after media is enabled and started is submitted locally; its promise never gates media.
|
|
74
|
+
* The started control frame precedes the provider.ready event frame. Earlier startup events may precede it.
|
|
75
|
+
*/
|
|
76
|
+
onStarted?(session: VoiceSession, context: VoiceBridgeStartedContext): void | Promise<void>;
|
|
77
|
+
/**
|
|
78
|
+
* Synchronous visibility policy for JSON event frames. Default: send every event.
|
|
79
|
+
* Does not filter playback configuration/interruption controls, binary audio, stored history, or server observers.
|
|
80
|
+
* Inspect without mutating the event; throwing closes the bridge with a generic client error.
|
|
81
|
+
*/
|
|
82
|
+
shouldSendEvent?(event: HarnessEvent): boolean;
|
|
83
|
+
/** Observes command/hook errors and socket send failures, including when wire reporting is impossible. */
|
|
84
|
+
onError?(error: Error): void;
|
|
85
|
+
/** Bounds waiting for pending creation on close. Integer 1–2147483647 ms; default: 5 seconds. */
|
|
86
|
+
startupCloseTimeoutMs?: number;
|
|
50
87
|
inputSampleRate?: number;
|
|
51
88
|
}
|
|
52
89
|
export declare const VOICE_BRIDGE_INPUT_SAMPLE_RATE = 24000;
|
|
@@ -57,8 +94,12 @@ export declare class VoiceSocketBridge {
|
|
|
57
94
|
handleText(text: string): Promise<void>;
|
|
58
95
|
/** Feed every binary frame from the client socket here. */
|
|
59
96
|
handleBinary(data: Uint8Array): Promise<void>;
|
|
60
|
-
/**
|
|
61
|
-
|
|
97
|
+
/**
|
|
98
|
+
* Closes an attached session gracefully, or cancels pending creation and waits up to
|
|
99
|
+
* startupCloseTimeoutMs. Repeated calls share one result. Unconfirmed startup is still
|
|
100
|
+
* cleaned up if it returns later; this result never claims provider finalization.
|
|
101
|
+
*/
|
|
102
|
+
handleClose(reason?: string): Promise<VoiceBridgeCloseResult>;
|
|
62
103
|
}
|
|
63
104
|
export declare function encodeVoiceAudioFrame(chunk: AudioChunk): ArrayBuffer;
|
|
64
105
|
/** Event payload for clients: audio chunks shrink to metadata, bytes stay server-side. */
|