xo-harness 0.1.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/dist/internal/harness/index.d.ts +3 -0
- package/dist/internal/harness/index.js +3 -0
- package/dist/internal/harness/message.d.ts +1 -2
- package/dist/internal/harness/message.js +45 -19
- package/dist/internal/harness/report-diff.d.ts +46 -0
- package/dist/internal/harness/report-diff.js +62 -0
- package/dist/internal/harness/report.d.ts +2 -0
- package/dist/internal/harness/report.js +10 -0
- package/dist/internal/harness/shadow.d.ts +9 -2
- package/dist/internal/harness/shadow.js +5 -1
- package/dist/internal/harness/task-supervisor.js +21 -6
- package/dist/internal/harness/tool-delivery.d.ts +17 -0
- package/dist/internal/harness/tool-delivery.js +53 -0
- package/dist/internal/harness/tool-policy.d.ts +52 -0
- package/dist/internal/harness/tool-policy.js +22 -0
- package/dist/internal/harness/tool-runtime.d.ts +4 -0
- package/dist/internal/harness/tool-runtime.js +82 -1
- package/dist/internal/harness/tools.d.ts +22 -1
- package/dist/internal/harness/voice-session.d.ts +9 -1
- package/dist/internal/harness/voice-session.js +22 -2
- package/dist/internal/harness/xo.d.ts +13 -0
- package/dist/internal/harness/xo.js +8 -0
- package/dist/internal/protocol/events.d.ts +72 -0
- package/dist/internal/protocol/events.js +14 -1
- package/dist/internal/protocol/provider.d.ts +84 -2
- package/dist/internal/protocol/provider.js +51 -3
- package/dist/internal/provider/grok-voice.d.ts +1 -0
- package/dist/internal/provider/grok-voice.js +4 -0
- package/dist/internal/provider/openai-realtime.d.ts +10 -1
- package/dist/internal/provider/openai-realtime.js +26 -1
- package/dist/internal/provider/realtime-session.d.ts +6 -0
- package/dist/internal/provider/realtime-session.js +269 -32
- package/dist/internal/provider-fake/replay-voice-provider.d.ts +6 -2
- package/dist/internal/provider-fake/replay-voice-provider.js +13 -3
- package/dist/internal/skills/index.d.ts +2 -0
- package/dist/internal/skills/index.js +2 -0
- package/dist/internal/skills/node.d.ts +7 -0
- package/dist/internal/skills/node.js +99 -0
- package/dist/internal/skills/skill.d.ts +17 -0
- package/dist/internal/skills/skill.js +42 -0
- package/dist/internal/skills/tools.d.ts +7 -0
- package/dist/internal/skills/tools.js +82 -0
- package/dist/internal/storage/memory.d.ts +2 -0
- package/dist/internal/storage/memory.js +1 -0
- package/dist/internal/tools-openai/delegate-conversation.d.ts +12 -0
- package/dist/internal/tools-openai/delegate-conversation.js +66 -0
- package/dist/internal/tools-openai/index.d.ts +24 -0
- package/dist/internal/tools-openai/index.js +119 -0
- package/dist/internal/tools-openai/responses.d.ts +31 -0
- package/dist/internal/tools-openai/responses.js +146 -0
- package/dist/skills-node.d.ts +1 -0
- package/dist/skills-node.js +1 -0
- package/dist/skills.d.ts +1 -0
- package/dist/skills.js +1 -0
- package/dist/storage-memory.d.ts +1 -0
- package/dist/storage-memory.js +1 -0
- package/dist/tools-openai.d.ts +1 -0
- package/dist/tools-openai.js +1 -0
- package/package.json +18 -1
package/README.md
CHANGED
|
@@ -29,7 +29,24 @@ Additional entry points are available for focused imports:
|
|
|
29
29
|
- `xo-harness/provider`
|
|
30
30
|
- `xo-harness/provider/workers`
|
|
31
31
|
- `xo-harness/storage`
|
|
32
|
+
- `xo-harness/storage/memory` (portable, without Node JSONL persistence)
|
|
33
|
+
- `xo-harness/skills` (catalogs, bundled skills, and on-demand reading tools)
|
|
34
|
+
- `xo-harness/skills/node` (filesystem discovery)
|
|
35
|
+
- `xo-harness/tools/openai` (web search and background reasoning with `createOpenAITools`)
|
|
32
36
|
- `xo-harness/testing`
|
|
33
37
|
|
|
38
|
+
Skills follow the standard `SKILL.md` format. Compose `createSkillTools(skills)` with your application
|
|
39
|
+
tools; reads use the existing admission policy and durable event log. See the repository's
|
|
40
|
+
[skills guide](https://github.com/zvadaadam/XO/blob/main/docs/skills.md) for usage and limits.
|
|
41
|
+
|
|
42
|
+
`createOpenAITools({ apiKey })` adds sourced `web_search` and non-blocking `delegate` tools. The worker
|
|
43
|
+
uses GPT-5.6 Terra with medium reasoning and reuses one worker conversation per XO session. Follow-ups
|
|
44
|
+
queue and remember earlier tasks and results. The harness reports status/results and aborts outstanding
|
|
45
|
+
work on session close. Voice and worker reasoning efforts are independent. See the
|
|
46
|
+
[tools guide](https://github.com/zvadaadam/XO/blob/main/docs/tools.md) for configuration and limits.
|
|
47
|
+
|
|
48
|
+
Tool results are delivered in full by default. `maxToolResultBytes` opts into string previews for
|
|
49
|
+
oversized results; finite values must be integers of at least 128 bytes, excluding provider envelopes.
|
|
50
|
+
|
|
34
51
|
See the [XO repository](https://github.com/zvadaadam/XO) for the complete documentation and voice
|
|
35
52
|
application.
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
export * from "./message.js";
|
|
2
2
|
export * from "./report.js";
|
|
3
|
+
export * from "./report-diff.js";
|
|
3
4
|
export * from "./shadow.js";
|
|
4
5
|
export * from "./socket-bridge.js";
|
|
6
|
+
export * from "./tool-delivery.js";
|
|
7
|
+
export * from "./tool-policy.js";
|
|
5
8
|
export * from "./tools.js";
|
|
6
9
|
export * from "./voice-session.js";
|
|
7
10
|
export * from "./xo.js";
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
export * from "./message.js";
|
|
2
2
|
export * from "./report.js";
|
|
3
|
+
export * from "./report-diff.js";
|
|
3
4
|
export * from "./shadow.js";
|
|
4
5
|
export * from "./socket-bridge.js";
|
|
6
|
+
export * from "./tool-delivery.js";
|
|
7
|
+
export * from "./tool-policy.js";
|
|
5
8
|
export * from "./tools.js";
|
|
6
9
|
export * from "./voice-session.js";
|
|
7
10
|
export * from "./xo.js";
|
|
@@ -9,8 +9,7 @@ import type { HarnessEvent, Message } from "../protocol/index.js";
|
|
|
9
9
|
* Folding rules:
|
|
10
10
|
* - `message.input` → a user message; its recorded `parts` project with full fidelity
|
|
11
11
|
* (image/file parts become `file` parts), a bare-text input becomes one text part.
|
|
12
|
-
* -
|
|
13
|
-
* - assistant `transcript.delta` → a `streaming` text part accumulating deltas,
|
|
12
|
+
* - `transcript.delta` from either speaker → a `streaming` text part accumulating deltas,
|
|
14
13
|
* finalized in place by the matching `transcript` (streamId-correlated, so a
|
|
15
14
|
* barge-in interrupted stream still finalizes into its original part).
|
|
16
15
|
* - `audio.output` → an `audio` part on the current assistant message (deduped per
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { toolDenialError } from "./tool-policy.js";
|
|
1
2
|
/**
|
|
2
3
|
* Projects the flat, sequence-ordered event log into the v2 `Message[]` part model —
|
|
3
4
|
* the shape every agent harness renders and persists (opencode / Vercel AI SDK v5 /
|
|
@@ -8,8 +9,7 @@
|
|
|
8
9
|
* Folding rules:
|
|
9
10
|
* - `message.input` → a user message; its recorded `parts` project with full fidelity
|
|
10
11
|
* (image/file parts become `file` parts), a bare-text input becomes one text part.
|
|
11
|
-
* -
|
|
12
|
-
* - assistant `transcript.delta` → a `streaming` text part accumulating deltas,
|
|
12
|
+
* - `transcript.delta` from either speaker → a `streaming` text part accumulating deltas,
|
|
13
13
|
* finalized in place by the matching `transcript` (streamId-correlated, so a
|
|
14
14
|
* barge-in interrupted stream still finalizes into its original part).
|
|
15
15
|
* - `audio.output` → an `audio` part on the current assistant message (deduped per
|
|
@@ -32,9 +32,9 @@ export function projectMessages(events) {
|
|
|
32
32
|
// streamId → the assistant message + audio part, so repeated chunks fold into one.
|
|
33
33
|
const audioParts = new Map();
|
|
34
34
|
// stream key → the streaming text part accumulating deltas, finalized by `transcript`.
|
|
35
|
-
// Deltas without a streamId key on the open assistant message, so a new message
|
|
36
|
-
// naturally starts a new default stream — no sentinel, no clearing.
|
|
37
35
|
const textParts = new Map();
|
|
36
|
+
// Speakers can overlap even when a provider omits stream IDs.
|
|
37
|
+
const streamlessTextKeys = new Map();
|
|
38
38
|
// Identity is the opening event: message id = event id, part id = event id + ordinal.
|
|
39
39
|
const partBase = (message, event, ordinal = 0) => ({
|
|
40
40
|
id: `${event.id}/p${ordinal}`,
|
|
@@ -69,15 +69,19 @@ export function projectMessages(events) {
|
|
|
69
69
|
break;
|
|
70
70
|
}
|
|
71
71
|
case "transcript.delta": {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
const
|
|
76
|
-
const existing = textParts.get(key);
|
|
72
|
+
const activeKey = event.streamId === undefined
|
|
73
|
+
? streamlessTextKeys.get(event.role)
|
|
74
|
+
: `${event.role}:${event.streamId}`;
|
|
75
|
+
const existing = activeKey === undefined ? undefined : textParts.get(activeKey);
|
|
77
76
|
if (existing) {
|
|
78
|
-
existing.
|
|
77
|
+
if (existing.state === "streaming")
|
|
78
|
+
existing.text += event.delta;
|
|
79
79
|
break;
|
|
80
80
|
}
|
|
81
|
+
const message = event.role === "assistant" ? assistantMessage(event) : openMessage("user", event);
|
|
82
|
+
const key = `${event.role}:${event.streamId ?? message.id}`;
|
|
83
|
+
if (event.streamId === undefined)
|
|
84
|
+
streamlessTextKeys.set(event.role, key);
|
|
81
85
|
const part = {
|
|
82
86
|
...partBase(message, event),
|
|
83
87
|
type: "text",
|
|
@@ -89,23 +93,31 @@ export function projectMessages(events) {
|
|
|
89
93
|
break;
|
|
90
94
|
}
|
|
91
95
|
case "transcript": {
|
|
92
|
-
if (event.role === "user") {
|
|
93
|
-
const message = openMessage("user", event);
|
|
94
|
-
message.parts.push({ ...partBase(message, event), type: "text", text: event.text, state: "done" });
|
|
95
|
-
break;
|
|
96
|
-
}
|
|
97
96
|
// Finalize the streaming part these deltas were building, if any — even when a
|
|
98
97
|
// barge-in already moved the conversation on — else record a fresh done part.
|
|
99
|
-
const key = event.streamId
|
|
98
|
+
const key = event.streamId === undefined
|
|
99
|
+
? streamlessTextKeys.get(event.role)
|
|
100
|
+
: `${event.role}:${event.streamId}`;
|
|
100
101
|
const streaming = key === undefined ? undefined : textParts.get(key);
|
|
101
102
|
if (key !== undefined && streaming !== undefined) {
|
|
102
103
|
streaming.text = event.text;
|
|
103
104
|
streaming.state = "done";
|
|
104
|
-
|
|
105
|
+
if (event.streamId === undefined) {
|
|
106
|
+
textParts.delete(key);
|
|
107
|
+
streamlessTextKeys.delete(event.role);
|
|
108
|
+
}
|
|
105
109
|
break;
|
|
106
110
|
}
|
|
107
|
-
const message = assistantMessage(event);
|
|
108
|
-
|
|
111
|
+
const message = event.role === "assistant" ? assistantMessage(event) : openMessage("user", event);
|
|
112
|
+
const part = {
|
|
113
|
+
...partBase(message, event),
|
|
114
|
+
type: "text",
|
|
115
|
+
text: event.text,
|
|
116
|
+
state: "done",
|
|
117
|
+
};
|
|
118
|
+
if (event.streamId !== undefined)
|
|
119
|
+
textParts.set(`${event.role}:${event.streamId}`, part);
|
|
120
|
+
message.parts.push(part);
|
|
109
121
|
break;
|
|
110
122
|
}
|
|
111
123
|
case "audio.output": {
|
|
@@ -165,6 +177,20 @@ export function projectMessages(events) {
|
|
|
165
177
|
};
|
|
166
178
|
break;
|
|
167
179
|
}
|
|
180
|
+
case "tool.denied": {
|
|
181
|
+
// Admission refusal settles the part like a failure; the log keeps the distinction.
|
|
182
|
+
const part = toolParts.get(event.callId);
|
|
183
|
+
if (part?.state.status !== "running")
|
|
184
|
+
break;
|
|
185
|
+
part.state = {
|
|
186
|
+
status: "settled",
|
|
187
|
+
input: part.state.input,
|
|
188
|
+
outcome: { type: "failed", error: toolDenialError(event.reason) },
|
|
189
|
+
startedAtMs: part.state.startedAtMs,
|
|
190
|
+
endedAtMs: event.recordedAtMs,
|
|
191
|
+
};
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
168
194
|
case "task.completed":
|
|
169
195
|
case "task.failed":
|
|
170
196
|
case "task.cancelled": {
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { SessionReport, UsageTotals } from "./report.js";
|
|
2
|
+
/** One metric compared across two reports; `delta` = candidate − baseline when both exist. */
|
|
3
|
+
export interface MetricDelta {
|
|
4
|
+
baseline?: number;
|
|
5
|
+
candidate?: number;
|
|
6
|
+
delta?: number;
|
|
7
|
+
}
|
|
8
|
+
/** Per-tool-name comparison; a denial counts as a failure (its outcome is the delivered one). */
|
|
9
|
+
export interface ToolCallDiff {
|
|
10
|
+
name: string;
|
|
11
|
+
baselineCalls: number;
|
|
12
|
+
candidateCalls: number;
|
|
13
|
+
baselineFailures: number;
|
|
14
|
+
candidateFailures: number;
|
|
15
|
+
meanLatencyMs: MetricDelta;
|
|
16
|
+
}
|
|
17
|
+
export interface SessionReportDiff {
|
|
18
|
+
baseline: {
|
|
19
|
+
sessionId: string;
|
|
20
|
+
providerId: string;
|
|
21
|
+
endedReason?: string;
|
|
22
|
+
};
|
|
23
|
+
candidate: {
|
|
24
|
+
sessionId: string;
|
|
25
|
+
providerId: string;
|
|
26
|
+
endedReason?: string;
|
|
27
|
+
};
|
|
28
|
+
durationMs: MetricDelta;
|
|
29
|
+
readyAtMs: MetricDelta;
|
|
30
|
+
responseLatencyMs: MetricDelta;
|
|
31
|
+
inputAudioMs: MetricDelta;
|
|
32
|
+
outputAudioMs: MetricDelta;
|
|
33
|
+
transcriptLines: MetricDelta;
|
|
34
|
+
errorCount: MetricDelta;
|
|
35
|
+
usage: Record<keyof UsageTotals, MetricDelta>;
|
|
36
|
+
/** Union of tool names from both reports, sorted, one row each. */
|
|
37
|
+
toolCalls: ToolCallDiff[];
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Compares two session reports metric by metric — typically a recorded baseline against
|
|
41
|
+
* a `runShadowSession` re-drive of the same input at another provider (or another build).
|
|
42
|
+
* Pure and deterministic over its inputs, like the reports themselves. Numbers only:
|
|
43
|
+
* judging transcript *content* needs a semantic oracle (e.g. re-transcribing the PCM),
|
|
44
|
+
* which stays outside the harness.
|
|
45
|
+
*/
|
|
46
|
+
export declare function diffSessionReports(baseline: SessionReport, candidate: SessionReport): SessionReportDiff;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Compares two session reports metric by metric — typically a recorded baseline against
|
|
3
|
+
* a `runShadowSession` re-drive of the same input at another provider (or another build).
|
|
4
|
+
* Pure and deterministic over its inputs, like the reports themselves. Numbers only:
|
|
5
|
+
* judging transcript *content* needs a semantic oracle (e.g. re-transcribing the PCM),
|
|
6
|
+
* which stays outside the harness.
|
|
7
|
+
*/
|
|
8
|
+
export function diffSessionReports(baseline, candidate) {
|
|
9
|
+
const usage = {};
|
|
10
|
+
for (const key of Object.keys(baseline.usage)) {
|
|
11
|
+
usage[key] = metric(baseline.usage[key], candidate.usage[key]);
|
|
12
|
+
}
|
|
13
|
+
const names = [...new Set([...baseline.toolCalls, ...candidate.toolCalls].map((call) => call.name))].sort();
|
|
14
|
+
return {
|
|
15
|
+
baseline: identity(baseline),
|
|
16
|
+
candidate: identity(candidate),
|
|
17
|
+
durationMs: metric(baseline.durationMs, candidate.durationMs),
|
|
18
|
+
readyAtMs: metric(baseline.readyAtMs, candidate.readyAtMs),
|
|
19
|
+
responseLatencyMs: metric(baseline.responseLatencyMs, candidate.responseLatencyMs),
|
|
20
|
+
inputAudioMs: metric(baseline.inputAudioMs, candidate.inputAudioMs),
|
|
21
|
+
outputAudioMs: metric(baseline.outputAudioMs, candidate.outputAudioMs),
|
|
22
|
+
transcriptLines: metric(baseline.transcripts.length, candidate.transcripts.length),
|
|
23
|
+
errorCount: metric(baseline.errors.length, candidate.errors.length),
|
|
24
|
+
usage,
|
|
25
|
+
toolCalls: names.map((name) => toolDiff(name, baseline.toolCalls, candidate.toolCalls)),
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function identity(report) {
|
|
29
|
+
return {
|
|
30
|
+
sessionId: report.sessionId,
|
|
31
|
+
providerId: report.providerId,
|
|
32
|
+
...(report.endedReason === undefined ? {} : { endedReason: report.endedReason }),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
function toolDiff(name, baseline, candidate) {
|
|
36
|
+
const base = baseline.filter((call) => call.name === name);
|
|
37
|
+
const cand = candidate.filter((call) => call.name === name);
|
|
38
|
+
return {
|
|
39
|
+
name,
|
|
40
|
+
baselineCalls: base.length,
|
|
41
|
+
candidateCalls: cand.length,
|
|
42
|
+
baselineFailures: failures(base),
|
|
43
|
+
candidateFailures: failures(cand),
|
|
44
|
+
meanLatencyMs: metric(meanLatency(base), meanLatency(cand)),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
function failures(calls) {
|
|
48
|
+
return calls.filter((call) => call.outcome?.type === "failed").length;
|
|
49
|
+
}
|
|
50
|
+
function meanLatency(calls) {
|
|
51
|
+
const latencies = calls.map((call) => call.latencyMs).filter((value) => value !== undefined);
|
|
52
|
+
if (latencies.length === 0)
|
|
53
|
+
return undefined;
|
|
54
|
+
return latencies.reduce((sum, value) => sum + value, 0) / latencies.length;
|
|
55
|
+
}
|
|
56
|
+
function metric(baseline, candidate) {
|
|
57
|
+
return {
|
|
58
|
+
...(baseline === undefined ? {} : { baseline }),
|
|
59
|
+
...(candidate === undefined ? {} : { candidate }),
|
|
60
|
+
...(baseline === undefined || candidate === undefined ? {} : { delta: candidate - baseline }),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
@@ -6,6 +6,8 @@ export interface ToolCallReport {
|
|
|
6
6
|
requestedAtMs: number;
|
|
7
7
|
outcome?: ToolOutcome;
|
|
8
8
|
latencyMs?: number;
|
|
9
|
+
/** Present when the admission policy refused the call; `outcome` is the delivered failure. */
|
|
10
|
+
deniedReason?: string;
|
|
9
11
|
}
|
|
10
12
|
export interface TranscriptLine {
|
|
11
13
|
role: "user" | "assistant";
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { audioSampleCount } from "../protocol/index.js";
|
|
2
|
+
import { toolDenialError } from "./tool-policy.js";
|
|
2
3
|
/**
|
|
3
4
|
* Derives latency and outcome metrics from a session log. Pure over the recorded events,
|
|
4
5
|
* so it works identically on live sessions, replays, and shadow runs.
|
|
@@ -80,6 +81,15 @@ export function summarizeSession(events) {
|
|
|
80
81
|
}
|
|
81
82
|
break;
|
|
82
83
|
}
|
|
84
|
+
case "tool.denied": {
|
|
85
|
+
const call = toolCalls.get(event.callId);
|
|
86
|
+
if (call) {
|
|
87
|
+
call.outcome = { type: "failed", error: toolDenialError(event.reason) };
|
|
88
|
+
call.latencyMs = event.sessionTimeMs - call.requestedAtMs;
|
|
89
|
+
call.deniedReason = event.reason;
|
|
90
|
+
}
|
|
91
|
+
break;
|
|
92
|
+
}
|
|
83
93
|
case "task.accepted":
|
|
84
94
|
taskIds.add(event.taskId);
|
|
85
95
|
break;
|
|
@@ -1,15 +1,20 @@
|
|
|
1
1
|
import type { HarnessEvent } from "../protocol/index.js";
|
|
2
2
|
import type { VoiceProvider } from "../provider/index.js";
|
|
3
|
-
import { type EventStore } from "../storage/
|
|
3
|
+
import { type EventStore } from "../storage/memory.js";
|
|
4
4
|
import { type SessionReport } from "./report.js";
|
|
5
|
+
import type { ToolPolicy } from "./tool-policy.js";
|
|
5
6
|
import type { VoiceTool } from "./tools.js";
|
|
6
7
|
export interface ShadowRunOptions {
|
|
7
8
|
/** The recorded session whose microphone audio is re-driven. */
|
|
8
9
|
events: readonly HarnessEvent[];
|
|
9
10
|
/** The provider to shadow-test with the recorded input. */
|
|
10
11
|
provider: VoiceProvider;
|
|
11
|
-
/** Tools to
|
|
12
|
+
/** Tools to execute during the shadow run; use substitutes when side effects are unwanted. */
|
|
12
13
|
tools?: readonly VoiceTool[];
|
|
14
|
+
/** Admission for this run. Defaults to allow-all; never inferred from the recorded log. */
|
|
15
|
+
toolPolicy?: ToolPolicy;
|
|
16
|
+
/** Byte budget for delivered results in this run; follows the normal session default. */
|
|
17
|
+
maxToolResultBytes?: number;
|
|
13
18
|
/** Where the shadow session records. Defaults to an in-memory store. */
|
|
14
19
|
store?: EventStore;
|
|
15
20
|
sessionId?: string;
|
|
@@ -22,6 +27,8 @@ export interface ShadowRunOptions {
|
|
|
22
27
|
/**
|
|
23
28
|
* The silent-test primitive: re-drives a recorded session's input audio against another
|
|
24
29
|
* provider and returns the resulting session report for comparison with the original.
|
|
30
|
+
* The shadow run executes supplied tools and applies only the policy provided here;
|
|
31
|
+
* a recording's denials do not establish permissions for a new session.
|
|
25
32
|
* Input is sent as fast as the provider accepts it; server-side VAD sees the recorded
|
|
26
33
|
* speech and trailing silence exactly as the original session did.
|
|
27
34
|
*/
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import { MemoryEventStore } from "../storage/
|
|
1
|
+
import { MemoryEventStore } from "../storage/memory.js";
|
|
2
2
|
import { summarizeSession } from "./report.js";
|
|
3
3
|
import { XO } from "./xo.js";
|
|
4
4
|
/**
|
|
5
5
|
* The silent-test primitive: re-drives a recorded session's input audio against another
|
|
6
6
|
* provider and returns the resulting session report for comparison with the original.
|
|
7
|
+
* The shadow run executes supplied tools and applies only the policy provided here;
|
|
8
|
+
* a recording's denials do not establish permissions for a new session.
|
|
7
9
|
* Input is sent as fast as the provider accepts it; server-side VAD sees the recorded
|
|
8
10
|
* speech and trailing silence exactly as the original session did.
|
|
9
11
|
*/
|
|
@@ -18,6 +20,8 @@ export async function runShadowSession(options) {
|
|
|
18
20
|
const harness = new XO({
|
|
19
21
|
store: options.store ?? new MemoryEventStore(),
|
|
20
22
|
...(options.tools === undefined ? {} : { tools: options.tools }),
|
|
23
|
+
...(options.toolPolicy === undefined ? {} : { toolPolicy: options.toolPolicy }),
|
|
24
|
+
...(options.maxToolResultBytes === undefined ? {} : { maxToolResultBytes: options.maxToolResultBytes }),
|
|
21
25
|
});
|
|
22
26
|
const session = await harness.startSession({
|
|
23
27
|
provider: options.provider,
|
|
@@ -16,7 +16,7 @@ export class TaskSupervisor {
|
|
|
16
16
|
}
|
|
17
17
|
forCall(callId) {
|
|
18
18
|
return {
|
|
19
|
-
start: (runner) => this.#register(callId, runner),
|
|
19
|
+
start: (runner, options) => this.#register(callId, runner, options),
|
|
20
20
|
};
|
|
21
21
|
}
|
|
22
22
|
async activate(taskId, callId) {
|
|
@@ -43,6 +43,7 @@ export class TaskSupervisor {
|
|
|
43
43
|
this.#tasks.delete(taskId);
|
|
44
44
|
task.detachSessionAbort();
|
|
45
45
|
task.controller.abort(reason);
|
|
46
|
+
this.#releasePending(task);
|
|
46
47
|
await this.#settle({ type: "task.cancelled", taskId, callId, reason }, false);
|
|
47
48
|
return true;
|
|
48
49
|
}
|
|
@@ -54,6 +55,7 @@ export class TaskSupervisor {
|
|
|
54
55
|
this.#tasks.delete(taskId);
|
|
55
56
|
task.detachSessionAbort();
|
|
56
57
|
task.controller.abort(reason);
|
|
58
|
+
this.#releasePending(task);
|
|
57
59
|
pendingCancellations.push(this.#settle({ type: "task.cancelled", taskId, callId: task.callId, reason }, true).then(() => undefined));
|
|
58
60
|
}
|
|
59
61
|
else {
|
|
@@ -64,27 +66,40 @@ export class TaskSupervisor {
|
|
|
64
66
|
}
|
|
65
67
|
await Promise.allSettled([...pendingCancellations, ...running]);
|
|
66
68
|
}
|
|
67
|
-
|
|
69
|
+
#releasePending(task) {
|
|
70
|
+
try {
|
|
71
|
+
task.onPendingCancel?.();
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
this.#onError(error);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
async #register(callId, runner, options) {
|
|
68
78
|
if (this.#sessionSignal.aborted)
|
|
69
79
|
throw new Error("Session is closing");
|
|
70
80
|
const taskId = crypto.randomUUID();
|
|
71
81
|
const controller = new AbortController();
|
|
72
82
|
const abortFromSession = () => controller.abort(this.#sessionSignal.reason);
|
|
73
83
|
this.#sessionSignal.addEventListener("abort", abortFromSession, { once: true });
|
|
74
|
-
|
|
84
|
+
const task = {
|
|
75
85
|
callId,
|
|
76
86
|
controller,
|
|
77
87
|
runner,
|
|
88
|
+
onPendingCancel: options?.onPendingCancel,
|
|
78
89
|
state: "pending",
|
|
79
90
|
detachSessionAbort: () => this.#sessionSignal.removeEventListener("abort", abortFromSession),
|
|
80
|
-
}
|
|
91
|
+
};
|
|
92
|
+
this.#tasks.set(taskId, task);
|
|
81
93
|
try {
|
|
82
94
|
await this.#record({ type: "task.accepted", taskId, callId });
|
|
83
95
|
}
|
|
84
96
|
catch (error) {
|
|
85
|
-
|
|
86
|
-
this.#
|
|
97
|
+
// Cancellation may already have released the reservation while recording waited.
|
|
98
|
+
const removed = this.#tasks.delete(taskId);
|
|
99
|
+
task.detachSessionAbort();
|
|
87
100
|
controller.abort("task_acceptance_failed");
|
|
101
|
+
if (removed)
|
|
102
|
+
this.#releasePending(task);
|
|
88
103
|
throw error;
|
|
89
104
|
}
|
|
90
105
|
return taskId;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { ToolOutcome } from "../protocol/index.js";
|
|
2
|
+
/**
|
|
3
|
+
* Results are delivered in full by default. Hosts can opt into a finite byte budget
|
|
4
|
+
* to limit model context; the durable log always keeps the full outcome.
|
|
5
|
+
*/
|
|
6
|
+
export declare const DEFAULT_MAX_TOOL_RESULT_BYTES: number;
|
|
7
|
+
/** Finite budgets must leave room for the preview marker and JSON encoding. */
|
|
8
|
+
export declare function validateMaxToolResultBytes(maxBytes: number): void;
|
|
9
|
+
/**
|
|
10
|
+
* Bounds the UTF-8 JSON encoding of a completed value or failure string, excluding
|
|
11
|
+
* provider-specific envelopes. Accepted task handles are unchanged. Finite budgets
|
|
12
|
+
* must be integers of at least 128 bytes; Infinity disables truncation.
|
|
13
|
+
*
|
|
14
|
+
* An oversized payload becomes a prefix of its JSON representation plus a marker
|
|
15
|
+
* identifying the source byte count. Only the provider-bound copy changes.
|
|
16
|
+
*/
|
|
17
|
+
export declare function boundToolOutcomeForDelivery(outcome: ToolOutcome, maxBytes: number): ToolOutcome;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Results are delivered in full by default. Hosts can opt into a finite byte budget
|
|
3
|
+
* to limit model context; the durable log always keeps the full outcome.
|
|
4
|
+
*/
|
|
5
|
+
export const DEFAULT_MAX_TOOL_RESULT_BYTES = Number.POSITIVE_INFINITY;
|
|
6
|
+
/** Finite budgets must leave room for the preview marker and JSON encoding. */
|
|
7
|
+
export function validateMaxToolResultBytes(maxBytes) {
|
|
8
|
+
if (maxBytes !== Number.POSITIVE_INFINITY && (!Number.isInteger(maxBytes) || maxBytes < 128)) {
|
|
9
|
+
throw new RangeError("maxToolResultBytes must be an integer of at least 128 bytes or Infinity");
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Bounds the UTF-8 JSON encoding of a completed value or failure string, excluding
|
|
14
|
+
* provider-specific envelopes. Accepted task handles are unchanged. Finite budgets
|
|
15
|
+
* must be integers of at least 128 bytes; Infinity disables truncation.
|
|
16
|
+
*
|
|
17
|
+
* An oversized payload becomes a prefix of its JSON representation plus a marker
|
|
18
|
+
* identifying the source byte count. Only the provider-bound copy changes.
|
|
19
|
+
*/
|
|
20
|
+
export function boundToolOutcomeForDelivery(outcome, maxBytes) {
|
|
21
|
+
validateMaxToolResultBytes(maxBytes);
|
|
22
|
+
if (maxBytes === Number.POSITIVE_INFINITY || outcome.type === "accepted_task")
|
|
23
|
+
return outcome;
|
|
24
|
+
const serialized = JSON.stringify(outcome.type === "completed" ? outcome.value : outcome.error);
|
|
25
|
+
const encoder = new TextEncoder();
|
|
26
|
+
const bytes = encoder.encode(serialized);
|
|
27
|
+
if (bytes.byteLength <= maxBytes)
|
|
28
|
+
return outcome;
|
|
29
|
+
// The preview is itself delivered as a JSON string. Include its escaping and the
|
|
30
|
+
// widest possible marker when finding a prefix, not just the source's raw bytes.
|
|
31
|
+
const suffix = marker(maxBytes, bytes.byteLength);
|
|
32
|
+
let low = 0;
|
|
33
|
+
let high = Math.min(maxBytes, bytes.byteLength);
|
|
34
|
+
while (low < high) {
|
|
35
|
+
const middle = Math.ceil((low + high) / 2);
|
|
36
|
+
const candidate = decodeUtf8Prefix(bytes, middle) + suffix;
|
|
37
|
+
if (encoder.encode(JSON.stringify(candidate)).byteLength <= maxBytes)
|
|
38
|
+
low = middle;
|
|
39
|
+
else
|
|
40
|
+
high = middle - 1;
|
|
41
|
+
}
|
|
42
|
+
const preview = decodeUtf8Prefix(bytes, low);
|
|
43
|
+
const text = preview + marker(encoder.encode(preview).byteLength, bytes.byteLength);
|
|
44
|
+
return outcome.type === "completed" ? { type: "completed", value: text } : { type: "failed", error: text };
|
|
45
|
+
}
|
|
46
|
+
function marker(shown, total) {
|
|
47
|
+
return `\n[truncated for delivery: showing the first ${shown} of ${total} bytes]`;
|
|
48
|
+
}
|
|
49
|
+
function decodeUtf8Prefix(bytes, byteLength) {
|
|
50
|
+
// Streaming decode withholds an incomplete trailing codepoint while preserving
|
|
51
|
+
// genuine U+FFFD characters already present in the source.
|
|
52
|
+
return new TextDecoder().decode(bytes.subarray(0, byteLength), { stream: true });
|
|
53
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { HarnessEvent, ToolCall } from "../protocol/index.js";
|
|
2
|
+
import type { ToolAnnotations } from "./tools.js";
|
|
3
|
+
/** Everything a policy may consult when admitting one tool call. */
|
|
4
|
+
export interface ToolAdmissionContext {
|
|
5
|
+
readonly call: ToolCall;
|
|
6
|
+
/** Behavior hints of the registered tool; absent when the call names an unknown tool. */
|
|
7
|
+
readonly annotations?: ToolAnnotations;
|
|
8
|
+
/**
|
|
9
|
+
* Aborts when the session closes. A policy that awaits external input — a client
|
|
10
|
+
* approval round-trip (e.g. ACP `session/request_permission`), a reviewer model —
|
|
11
|
+
* must abandon its work when this fires; the runtime stops waiting either way.
|
|
12
|
+
*/
|
|
13
|
+
readonly signal: AbortSignal;
|
|
14
|
+
/**
|
|
15
|
+
* The durable session log so far — e.g. transcript evidence that the user asked for
|
|
16
|
+
* this. Reads the store (audio events included), so call it only when a decision
|
|
17
|
+
* needs it, and project with `projectMessages` rather than scanning raw events.
|
|
18
|
+
*/
|
|
19
|
+
history(): Promise<readonly HarnessEvent[]>;
|
|
20
|
+
}
|
|
21
|
+
export type ToolAdmission = {
|
|
22
|
+
decision: "allow";
|
|
23
|
+
} | {
|
|
24
|
+
decision: "deny";
|
|
25
|
+
reason: string;
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* Admits or refuses model-requested tool calls before execution begins. A refusal is
|
|
29
|
+
* not a dead end: it is recorded as `tool.denied` and delivered to the provider as a
|
|
30
|
+
* failed outcome carrying the reason, so the model can revise its plan or ask the user
|
|
31
|
+
* out loud — in a voice session, permission conversations happen in the conversation.
|
|
32
|
+
* A policy that throws or returns a malformed decision refuses the call (the gate
|
|
33
|
+
* fails closed). Denials need a nonblank reason. Policies decide admission only;
|
|
34
|
+
* argument validation and execution stay in the tool runtime.
|
|
35
|
+
*/
|
|
36
|
+
export interface ToolPolicy {
|
|
37
|
+
admit(context: ToolAdmissionContext): ToolAdmission | Promise<ToolAdmission>;
|
|
38
|
+
}
|
|
39
|
+
/** The default policy: every requested tool executes. Preserves pre-policy behavior. */
|
|
40
|
+
export declare const allowAllToolPolicy: ToolPolicy;
|
|
41
|
+
export interface ToolPolicyRules {
|
|
42
|
+
/** Tool names that always execute. */
|
|
43
|
+
allow?: readonly string[];
|
|
44
|
+
/** Tool names that are always refused. A name on both lists is refused. */
|
|
45
|
+
deny?: readonly string[];
|
|
46
|
+
/** Decision for names on neither list. Default: "allow". */
|
|
47
|
+
otherwise?: "allow" | "deny";
|
|
48
|
+
}
|
|
49
|
+
/** A deterministic name-list policy; richer policies implement `ToolPolicy` directly. */
|
|
50
|
+
export declare function toolPolicyFromRules(rules: ToolPolicyRules): ToolPolicy;
|
|
51
|
+
/** The canonical rendering of a denial as a failed outcome, shared by delivery and projections. */
|
|
52
|
+
export declare function toolDenialError(reason: string): string;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/** The default policy: every requested tool executes. Preserves pre-policy behavior. */
|
|
2
|
+
export const allowAllToolPolicy = { admit: () => ({ decision: "allow" }) };
|
|
3
|
+
/** A deterministic name-list policy; richer policies implement `ToolPolicy` directly. */
|
|
4
|
+
export function toolPolicyFromRules(rules) {
|
|
5
|
+
const allow = new Set(rules.allow ?? []);
|
|
6
|
+
const deny = new Set(rules.deny ?? []);
|
|
7
|
+
const otherwise = rules.otherwise ?? "allow";
|
|
8
|
+
return {
|
|
9
|
+
admit: ({ call }) => {
|
|
10
|
+
if (deny.has(call.name)) {
|
|
11
|
+
return { decision: "deny", reason: `Tool "${call.name}" is denied by session policy.` };
|
|
12
|
+
}
|
|
13
|
+
if (allow.has(call.name) || otherwise === "allow")
|
|
14
|
+
return { decision: "allow" };
|
|
15
|
+
return { decision: "deny", reason: `Tool "${call.name}" is not on the session's allow list.` };
|
|
16
|
+
},
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
/** The canonical rendering of a denial as a failed outcome, shared by delivery and projections. */
|
|
20
|
+
export function toolDenialError(reason) {
|
|
21
|
+
return `Tool call denied: ${reason}`;
|
|
22
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type HarnessEvent, type NewHarnessEvent, type ProviderToolResult, type ToolCall } from "../protocol/index.js";
|
|
2
2
|
import type { TaskSupervisor } from "./task-supervisor.js";
|
|
3
|
+
import { type ToolPolicy } from "./tool-policy.js";
|
|
3
4
|
import type { ToolRegistry } from "./tools.js";
|
|
4
5
|
type RecordEvent = (event: NewHarnessEvent) => Promise<HarnessEvent>;
|
|
5
6
|
export interface ToolResultSink {
|
|
@@ -14,6 +15,9 @@ export declare class ToolRuntime {
|
|
|
14
15
|
resultSink: ToolResultSink;
|
|
15
16
|
signal: AbortSignal;
|
|
16
17
|
flush: () => Promise<void>;
|
|
18
|
+
policy: ToolPolicy;
|
|
19
|
+
history: () => Promise<readonly HarnessEvent[]>;
|
|
20
|
+
maxResultBytes: number;
|
|
17
21
|
});
|
|
18
22
|
execute(call: ToolCall): Promise<void>;
|
|
19
23
|
}
|