sdd-engine 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +165 -2
- package/dist/engine/cli.js +95 -0
- package/dist/engine/simulate.js +214 -0
- package/dist/judge/conversation.js +45 -0
- package/dist/judge/judge.js +122 -0
- package/dist/judge/outcome.js +42 -0
- package/dist/judge/regrade.js +74 -0
- package/dist/pipeline/agent.js +78 -0
- package/dist/pipeline/agentstage.js +88 -0
- package/dist/pipeline/approve.js +68 -0
- package/dist/pipeline/cli.js +261 -0
- package/dist/pipeline/derive.js +287 -0
- package/dist/pipeline/designloop.js +385 -0
- package/dist/pipeline/facts.js +108 -0
- package/dist/pipeline/mock.js +253 -0
- package/dist/pipeline/project.js +79 -0
- package/dist/pipeline/render.js +95 -0
- package/dist/pipeline/spec.js +272 -0
- package/dist/pipeline/try.js +162 -0
- package/dist/runner/batch-lib.js +157 -0
- package/dist/runner/batch.js +56 -0
- package/dist/runner/cli.js +61 -0
- package/dist/runner/llm/claude-code.js +123 -0
- package/dist/runner/llm/index.js +12 -0
- package/dist/runner/llm/types.js +17 -0
- package/dist/runner/reader/answer.js +76 -0
- package/dist/runner/reader/extract.js +106 -0
- package/dist/runner/reader/run.js +111 -0
- package/dist/runner/reader/skim.js +115 -0
- package/dist/runner/rng.js +22 -0
- package/dist/runner/sandbox/cli.js +60 -0
- package/dist/runner/sandbox/sandbox.js +144 -0
- package/dist/runner/sandbox/server.js +64 -0
- package/dist/runner/spec.js +84 -0
- package/dist/runner/talker/decide.js +110 -0
- package/dist/runner/talker/run.js +159 -0
- package/dist/runner/talker/world.js +36 -0
- package/dist/runner/trace.js +130 -0
- package/dist/runner/visitor/decide.js +159 -0
- package/dist/runner/visitor/observe.js +102 -0
- package/dist/runner/visitor/run.js +296 -0
- package/dist/scorer/compare.js +119 -0
- package/dist/scorer/score.js +201 -0
- package/dist/scorer/spot-check-compare.js +41 -0
- package/dist/scorer/spot-check-view.js +45 -0
- package/dist/scorer/stats.js +45 -0
- package/dist/scorer/summary.js +62 -0
- package/package.json +49 -4
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// Sandbox command line.
|
|
2
|
+
//
|
|
3
|
+
// node runner/sandbox/cli.ts serve [dir] [port] serve the page for a human to look at
|
|
4
|
+
// node runner/sandbox/cli.ts check [dir] load it in the guarded browser at each
|
|
5
|
+
// viewport, exit 1 on any outside request
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import { devices } from 'playwright';
|
|
8
|
+
import { startStaticServer } from "./server.js";
|
|
9
|
+
import { startSandbox } from "./sandbox.js";
|
|
10
|
+
const [command = 'check', dirArg, portArg, pageArg] = process.argv.slice(2);
|
|
11
|
+
const rootDir = path.resolve(dirArg ?? path.join(import.meta.dirname, '..', '..', 'site'));
|
|
12
|
+
const VIEWPORTS = {
|
|
13
|
+
mobile: devices['iPhone 13'],
|
|
14
|
+
desktop: { viewport: { width: 1280, height: 800 } },
|
|
15
|
+
};
|
|
16
|
+
if (command === 'serve') {
|
|
17
|
+
// An optional fourth argument serves a page variant at / with this directory's assets.
|
|
18
|
+
const server = await startStaticServer(rootDir, Number(portArg ?? 4173), pageArg ? path.resolve(pageArg) : undefined);
|
|
19
|
+
if (pageArg)
|
|
20
|
+
console.log(`Page at /: ${pageArg}`);
|
|
21
|
+
console.log(`Serving ${rootDir}`);
|
|
22
|
+
console.log(` ${server.origin}/`);
|
|
23
|
+
console.log('This is a plain server for looking at the page. Persona runs use the guarded browser.');
|
|
24
|
+
}
|
|
25
|
+
else if (command === 'check') {
|
|
26
|
+
const sandbox = await startSandbox({ rootDir });
|
|
27
|
+
let failed = false;
|
|
28
|
+
try {
|
|
29
|
+
for (const [name, options] of Object.entries(VIEWPORTS)) {
|
|
30
|
+
const context = await sandbox.newContext(options);
|
|
31
|
+
const page = await context.newPage();
|
|
32
|
+
const response = await page.goto(`${sandbox.origin}/`, { waitUntil: 'load' });
|
|
33
|
+
await page.evaluate(() => document.fonts.ready);
|
|
34
|
+
const words = (await page.innerText('body')).split(/\s+/).filter(Boolean).length;
|
|
35
|
+
const status = response?.status();
|
|
36
|
+
console.log(`${name}: HTTP ${status}, "${await page.title()}", ${words} visible words`);
|
|
37
|
+
if (status !== 200)
|
|
38
|
+
failed = true;
|
|
39
|
+
await context.close();
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
finally {
|
|
43
|
+
await sandbox.close();
|
|
44
|
+
}
|
|
45
|
+
if (sandbox.violations.length > 0) {
|
|
46
|
+
console.error(`FAIL: ${sandbox.violations.length} outside request(s) attempted`);
|
|
47
|
+
for (const v of sandbox.violations)
|
|
48
|
+
console.error(` [${v.layer}] ${v.url} (${v.detail})`);
|
|
49
|
+
process.exit(1);
|
|
50
|
+
}
|
|
51
|
+
if (failed) {
|
|
52
|
+
console.error('FAIL: page did not load');
|
|
53
|
+
process.exit(1);
|
|
54
|
+
}
|
|
55
|
+
console.log('OK: no outside requests attempted');
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
console.error(`Unknown command "${command}". Use serve or check.`);
|
|
59
|
+
process.exit(2);
|
|
60
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
// The sandbox: a local server for the page under test plus a browser that
|
|
2
|
+
// cannot reach anything else. Enforces hard rule 1 of the brief.
|
|
3
|
+
//
|
|
4
|
+
// Two independent layers:
|
|
5
|
+
//
|
|
6
|
+
// 1. Route guard. Every request in the browser context is checked. Anything
|
|
7
|
+
// that is not the sandbox server's own origin is aborted and recorded.
|
|
8
|
+
// WebSockets get the same treatment. Service workers are disabled.
|
|
9
|
+
//
|
|
10
|
+
// 2. Blackhole proxy. The browser is launched with a proxy that refuses
|
|
11
|
+
// everything and records what it was asked for. Chromium bypasses the
|
|
12
|
+
// proxy for loopback addresses, so the local server still works. This
|
|
13
|
+
// catches anything layer 1 cannot see or cannot stop.
|
|
14
|
+
//
|
|
15
|
+
// Any recorded violation fails the run: finish() throws.
|
|
16
|
+
import { createServer } from 'node:http';
|
|
17
|
+
import { chromium } from 'playwright';
|
|
18
|
+
import { closeServer, listen, startStaticServer } from "./server.js";
|
|
19
|
+
export class SandboxViolationError extends Error {
|
|
20
|
+
violations;
|
|
21
|
+
constructor(violations) {
|
|
22
|
+
const lines = violations.map((v) => ` [${v.layer}] ${v.url} (${v.detail})`);
|
|
23
|
+
super(`Sandbox violation: ${violations.length} outside request(s) attempted\n${lines.join('\n')}`);
|
|
24
|
+
this.name = 'SandboxViolationError';
|
|
25
|
+
this.violations = violations;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/** Schemes that never touch the network. */
|
|
29
|
+
const INERT = new Set(['data:', 'blob:', 'about:']);
|
|
30
|
+
export function isAllowedUrl(url, allowedOrigin) {
|
|
31
|
+
let parsed;
|
|
32
|
+
try {
|
|
33
|
+
parsed = new URL(url);
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
if (INERT.has(parsed.protocol))
|
|
39
|
+
return true;
|
|
40
|
+
return parsed.origin === allowedOrigin;
|
|
41
|
+
}
|
|
42
|
+
export async function startSandbox(options) {
|
|
43
|
+
const violations = [];
|
|
44
|
+
let site;
|
|
45
|
+
if (options.targetOrigin) {
|
|
46
|
+
const target = new URL(options.targetOrigin);
|
|
47
|
+
if (!['localhost', '127.0.0.1', '[::1]'].includes(target.hostname) || !/^https?:$/.test(target.protocol)) {
|
|
48
|
+
throw new Error(`Sandbox target must be a loopback http address, got ${options.targetOrigin}`);
|
|
49
|
+
}
|
|
50
|
+
site = { origin: target.origin, close: async () => { } };
|
|
51
|
+
}
|
|
52
|
+
else if (options.rootDir) {
|
|
53
|
+
site = await startStaticServer(options.rootDir, options.port, options.indexFile);
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
throw new Error('Sandbox needs rootDir or targetOrigin');
|
|
57
|
+
}
|
|
58
|
+
const origin = site.origin;
|
|
59
|
+
const wsOrigin = origin.replace(/^http/, 'ws');
|
|
60
|
+
// Layer 2: the blackhole proxy.
|
|
61
|
+
const blackhole = createServer((req, res) => {
|
|
62
|
+
violations.push({ layer: 'proxy', url: req.url ?? '', detail: `${req.method} reached the proxy` });
|
|
63
|
+
res.writeHead(502).end();
|
|
64
|
+
});
|
|
65
|
+
blackhole.on('connect', (req, socket) => {
|
|
66
|
+
violations.push({ layer: 'proxy', url: req.url ?? '', detail: 'CONNECT reached the proxy' });
|
|
67
|
+
socket.end('HTTP/1.1 502 Bad Gateway\r\n\r\n');
|
|
68
|
+
});
|
|
69
|
+
const proxyPort = await listen(blackhole);
|
|
70
|
+
let browser;
|
|
71
|
+
try {
|
|
72
|
+
browser = await chromium.launch({
|
|
73
|
+
// Raw flag on purpose. Playwright's own proxy option forces loopback
|
|
74
|
+
// traffic through the proxy too, which would break the local server.
|
|
75
|
+
args: [`--proxy-server=http://127.0.0.1:${proxyPort}`],
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
catch (err) {
|
|
79
|
+
await site.close();
|
|
80
|
+
await closeServer(blackhole);
|
|
81
|
+
throw err;
|
|
82
|
+
}
|
|
83
|
+
async function newContext(contextOptions = {}) {
|
|
84
|
+
const context = await browser.newContext({ ...contextOptions, serviceWorkers: 'block' });
|
|
85
|
+
if (options.unsafeDisableRouteGuard)
|
|
86
|
+
return context;
|
|
87
|
+
// Layer 1: abort and record anything that is not the sandbox origin.
|
|
88
|
+
await context.route('**/*', async (route) => {
|
|
89
|
+
const request = route.request();
|
|
90
|
+
const url = request.url();
|
|
91
|
+
if (isAllowedUrl(url, origin)) {
|
|
92
|
+
await route.continue();
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
violations.push({ layer: 'route', url, detail: `${request.method()} ${request.resourceType()} blocked` });
|
|
96
|
+
await route.abort('blockedbyclient');
|
|
97
|
+
});
|
|
98
|
+
await context.routeWebSocket(/.*/, async (ws) => {
|
|
99
|
+
const url = ws.url();
|
|
100
|
+
if (isAllowedUrl(url, wsOrigin)) {
|
|
101
|
+
ws.connectToServer();
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
violations.push({ layer: 'websocket', url, detail: 'websocket blocked' });
|
|
105
|
+
await ws.close({ code: 1008, reason: 'blocked by sandbox' });
|
|
106
|
+
});
|
|
107
|
+
// Routes do not see redirect hops. This event does, so a local URL that
|
|
108
|
+
// redirects outside still fails the run. Layer 2 is what stops it.
|
|
109
|
+
context.on('request', (request) => {
|
|
110
|
+
const url = request.url();
|
|
111
|
+
if (isAllowedUrl(url, origin))
|
|
112
|
+
return;
|
|
113
|
+
if (violations.some((v) => v.url === url))
|
|
114
|
+
return;
|
|
115
|
+
violations.push({ layer: 'request-event', url, detail: `${request.method()} ${request.resourceType()} seen` });
|
|
116
|
+
});
|
|
117
|
+
return context;
|
|
118
|
+
}
|
|
119
|
+
function assertClean() {
|
|
120
|
+
if (options.outsideRequests === 'block')
|
|
121
|
+
return;
|
|
122
|
+
if (violations.length > 0)
|
|
123
|
+
throw new SandboxViolationError([...violations]);
|
|
124
|
+
}
|
|
125
|
+
async function close() {
|
|
126
|
+
await browser.close();
|
|
127
|
+
await site.close();
|
|
128
|
+
await closeServer(blackhole);
|
|
129
|
+
}
|
|
130
|
+
return {
|
|
131
|
+
origin,
|
|
132
|
+
violations,
|
|
133
|
+
record: () => options.outsideRequests === 'block'
|
|
134
|
+
? { origin, violations: 0, blocked: violations.length }
|
|
135
|
+
: { origin, violations: violations.length },
|
|
136
|
+
newContext,
|
|
137
|
+
assertClean,
|
|
138
|
+
close,
|
|
139
|
+
async finish() {
|
|
140
|
+
await close();
|
|
141
|
+
assertClean();
|
|
142
|
+
},
|
|
143
|
+
};
|
|
144
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// Static file server for the page under test. Binds to 127.0.0.1 only,
|
|
2
|
+
// on an ephemeral port unless one is given.
|
|
3
|
+
import { createServer } from 'node:http';
|
|
4
|
+
import { readFile, stat } from 'node:fs/promises';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
const TYPES = {
|
|
7
|
+
'.html': 'text/html; charset=utf-8',
|
|
8
|
+
'.css': 'text/css; charset=utf-8',
|
|
9
|
+
'.js': 'text/javascript; charset=utf-8',
|
|
10
|
+
'.json': 'application/json',
|
|
11
|
+
'.svg': 'image/svg+xml',
|
|
12
|
+
'.png': 'image/png',
|
|
13
|
+
'.jpg': 'image/jpeg',
|
|
14
|
+
'.webp': 'image/webp',
|
|
15
|
+
'.ico': 'image/x-icon',
|
|
16
|
+
'.woff2': 'font/woff2',
|
|
17
|
+
};
|
|
18
|
+
export function listen(server, port = 0) {
|
|
19
|
+
return new Promise((resolve, reject) => {
|
|
20
|
+
server.once('error', reject);
|
|
21
|
+
server.listen(port, '127.0.0.1', () => resolve(server.address().port));
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
export function closeServer(server) {
|
|
25
|
+
return new Promise((resolve) => {
|
|
26
|
+
server.close(() => resolve());
|
|
27
|
+
server.closeAllConnections();
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Serves rootDir. If indexFile is given, that file is served at / in place of
|
|
32
|
+
* rootDir/index.html, so a page variant can be tested with the site's assets.
|
|
33
|
+
*/
|
|
34
|
+
export async function startStaticServer(rootDir, port = 0, indexFile) {
|
|
35
|
+
const root = path.resolve(rootDir);
|
|
36
|
+
const server = createServer(async (req, res) => {
|
|
37
|
+
try {
|
|
38
|
+
const pathname = decodeURIComponent(new URL(req.url ?? '/', 'http://x').pathname);
|
|
39
|
+
let file = path.join(root, pathname);
|
|
40
|
+
if (file !== root && !file.startsWith(root + path.sep)) {
|
|
41
|
+
res.writeHead(403).end('forbidden');
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
if (indexFile && (pathname === '/' || pathname === '/index.html'))
|
|
45
|
+
file = path.resolve(indexFile);
|
|
46
|
+
else if ((await stat(file)).isDirectory())
|
|
47
|
+
file = path.join(file, 'index.html');
|
|
48
|
+
const body = await readFile(file);
|
|
49
|
+
res.writeHead(200, {
|
|
50
|
+
'content-type': TYPES[path.extname(file)] ?? 'application/octet-stream',
|
|
51
|
+
'cache-control': 'no-store',
|
|
52
|
+
});
|
|
53
|
+
res.end(body);
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
res.writeHead(404).end('not found');
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
const boundPort = await listen(server, port);
|
|
60
|
+
return {
|
|
61
|
+
origin: `http://127.0.0.1:${boundPort}`,
|
|
62
|
+
close: () => closeServer(server),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
// Loads specs and personas. Read-only: nothing in the runner writes to them.
|
|
2
|
+
import { readFile } from 'node:fs/promises';
|
|
3
|
+
import { parse } from 'yaml';
|
|
4
|
+
export async function loadSpec(file) {
|
|
5
|
+
const spec = parse(await readFile(file, 'utf8'));
|
|
6
|
+
if (!spec.experiment)
|
|
7
|
+
throw new Error(`${file}: missing "experiment"`);
|
|
8
|
+
if (spec.kind === 'task') {
|
|
9
|
+
if (!spec.task?.goal || !spec.task.success)
|
|
10
|
+
throw new Error(`${file}: a task spec needs task.goal and task.success`);
|
|
11
|
+
const s = spec.task.success;
|
|
12
|
+
for (const key of ['leaves_to', 'clicks']) {
|
|
13
|
+
if (s[key] !== undefined && !(Array.isArray(s[key]) && s[key].length > 0 && s[key].every((v) => typeof v === 'string' && v.trim().length > 0))) {
|
|
14
|
+
throw new Error(`${file}: task.success.${key} must be a list of text`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
if (!s.request?.path_includes && !s.url_includes && !s.text_visible && !s.outcome && !s.leaves_to && !s.clicks) {
|
|
18
|
+
throw new Error(`${file}: task.success needs request, url_includes, text_visible, leaves_to, clicks, or outcome`);
|
|
19
|
+
}
|
|
20
|
+
if (!(spec.visitor && spec.visitor.patience_seconds > 0))
|
|
21
|
+
throw new Error(`${file}: a task spec needs visitor.patience_seconds`);
|
|
22
|
+
// One synthetic question, so thresholds (pass.task_rate), scoring, and reports work unchanged.
|
|
23
|
+
spec.questions = [{ id: 'task', ask: spec.task.goal, rubric: { required: [`the visitor completed: ${spec.task.goal}`] } }];
|
|
24
|
+
spec.reader ??= { time_budget_seconds: spec.visitor.patience_seconds, answers_from: 'what_was_seen', may_scroll_back: true };
|
|
25
|
+
}
|
|
26
|
+
if (spec.kind === 'conversation') {
|
|
27
|
+
if (!spec.task?.goal || !spec.task.success)
|
|
28
|
+
throw new Error(`${file}: a conversation spec needs task.goal and task.success`);
|
|
29
|
+
const s = spec.task.success;
|
|
30
|
+
if (!s.tool_called && !s.outcome)
|
|
31
|
+
throw new Error(`${file}: task.success needs tool_called, outcome, or both`);
|
|
32
|
+
if (s.request || s.url_includes || s.text_visible || s.leaves_to || s.clicks)
|
|
33
|
+
throw new Error(`${file}: a conversation has no screen, so task.success takes only tool_called and outcome`);
|
|
34
|
+
if (!(spec.talker && spec.talker.patience_seconds > 0 && spec.talker.adapter))
|
|
35
|
+
throw new Error(`${file}: a conversation spec needs talker.patience_seconds and talker.adapter`);
|
|
36
|
+
spec.questions = [{ id: 'task', ask: spec.task.goal, rubric: { required: [`the person got: ${spec.task.goal}`] } }];
|
|
37
|
+
spec.reader ??= { time_budget_seconds: spec.talker.patience_seconds, answers_from: 'what_was_heard', may_scroll_back: false };
|
|
38
|
+
spec.entry = { ...spec.entry, viewports: ['voice'] };
|
|
39
|
+
}
|
|
40
|
+
for (const key of ['questions', 'reader']) {
|
|
41
|
+
if (!spec[key])
|
|
42
|
+
throw new Error(`${file}: missing "${key}"`);
|
|
43
|
+
}
|
|
44
|
+
for (const question of spec.questions) {
|
|
45
|
+
question.rubric.required = rubricLines(file, question.id, question.rubric.required);
|
|
46
|
+
if (question.rubric.bonus)
|
|
47
|
+
question.rubric.bonus = rubricLines(file, question.id, question.rubric.bonus);
|
|
48
|
+
if (question.rubric.required.length === 0)
|
|
49
|
+
throw new Error(`${file}: question "${question.id}" has no required rubric elements`);
|
|
50
|
+
}
|
|
51
|
+
return spec;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Every rubric element must reach the judge as plain text. YAML reads a line
|
|
55
|
+
* like "- at least 3 of: a, b, c" as a one-key map, not a string, so that
|
|
56
|
+
* shape is joined back into the line as written. Anything else is an error.
|
|
57
|
+
*/
|
|
58
|
+
function rubricLines(file, questionId, items) {
|
|
59
|
+
if (!Array.isArray(items))
|
|
60
|
+
throw new Error(`${file}: rubric for "${questionId}" is not a list`);
|
|
61
|
+
return items.map((item) => {
|
|
62
|
+
if (typeof item === 'string')
|
|
63
|
+
return item;
|
|
64
|
+
if (item && typeof item === 'object' && !Array.isArray(item)) {
|
|
65
|
+
const entries = Object.entries(item);
|
|
66
|
+
if (entries.length === 1 && typeof entries[0][1] === 'string')
|
|
67
|
+
return `${entries[0][0]}: ${entries[0][1]}`;
|
|
68
|
+
}
|
|
69
|
+
throw new Error(`${file}: rubric element for "${questionId}" is not text: ${JSON.stringify(item)}`);
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
export async function loadPersona(file) {
|
|
73
|
+
const persona = parse(await readFile(file, 'utf8'));
|
|
74
|
+
for (const key of ['id', 'profile', 'arrival', 'needs', 'objections', 'dials']) {
|
|
75
|
+
if (!persona[key])
|
|
76
|
+
throw new Error(`${file}: missing "${key}"`);
|
|
77
|
+
}
|
|
78
|
+
const { reading_wpm, skim_rate } = persona.dials;
|
|
79
|
+
if (!(reading_wpm > 0))
|
|
80
|
+
throw new Error(`${file}: dials.reading_wpm must be positive`);
|
|
81
|
+
if (!(skim_rate >= 0 && skim_rate <= 1))
|
|
82
|
+
throw new Error(`${file}: dials.skim_rate must be 0 to 1`);
|
|
83
|
+
return persona;
|
|
84
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// The talker's next line. One model call per turn.
|
|
2
|
+
//
|
|
3
|
+
// Same stance as the visitor (see ../visitor/decide.ts): the model predicts
|
|
4
|
+
// what a described person does, it is not asked to succeed, and it never
|
|
5
|
+
// learns what counts as success. After the first line, anything other than
|
|
6
|
+
// giving up must quote words the product actually said, and the engine
|
|
7
|
+
// checks the quote. A person who heard nothing usable has no reason to stay.
|
|
8
|
+
import { citeIsOnScreen } from "../visitor/decide.js";
|
|
9
|
+
const SYSTEM = `You predict what one specific person says next while talking out loud to a voice assistant or robot. You are not that person and you are not trying to help them or the product. You are forecasting behavior.
|
|
10
|
+
|
|
11
|
+
You get a description of the person, what they want, and the conversation so far: what they said and what they heard back.
|
|
12
|
+
|
|
13
|
+
Choose exactly one next action:
|
|
14
|
+
- "say": they say something. Put their exact spoken words in "text".
|
|
15
|
+
- "done": they believe they got what they came for, and stop talking.
|
|
16
|
+
- "leave": they give up and walk away.
|
|
17
|
+
|
|
18
|
+
Rules:
|
|
19
|
+
- Write "text" the way this person actually talks out loud: short, informal, sometimes vague or incomplete, in their own words. No stage directions, no quotation marks, nothing they would not say aloud.
|
|
20
|
+
- They do not know what the product can do or how it wants to be asked. They ask the way they would ask a person.
|
|
21
|
+
- People give up when they are misunderstood twice, when they get a speech where they wanted an action, when they are asked questions they did not expect and cannot be bothered to answer, or when the reply does not match what they asked.
|
|
22
|
+
- People do not repeat themselves more than once or twice. A third try is rare.
|
|
23
|
+
- After the first line, to choose "say" or "done" you must quote in "cites" the exact words they heard that they are responding to. For "done", quote the words that tell them it worked. People who hear no confirmation are not sure it worked.
|
|
24
|
+
- For "leave", put in "cites" the words that put them off, or an empty string if they heard nothing useful.
|
|
25
|
+
- "reason" is one sentence in the third person saying why this person does this.`;
|
|
26
|
+
export function buildTalkerRequest(input) {
|
|
27
|
+
const { persona } = input;
|
|
28
|
+
const first = input.exchanges.length === 0;
|
|
29
|
+
const prompt = `THE PERSON
|
|
30
|
+
${persona.archetype}
|
|
31
|
+
Role: ${persona.profile.role}
|
|
32
|
+
Experience: ${persona.profile.experience}
|
|
33
|
+
Already uses: ${persona.profile.already_uses.join('; ')}
|
|
34
|
+
${persona.speaks ? `How they talk: ${persona.speaks}\n` : ''}Hoping for: ${persona.needs.join('; ')}
|
|
35
|
+
Doubts going in: ${persona.objections.join('; ')}
|
|
36
|
+
|
|
37
|
+
WHAT THEY ARE DOING
|
|
38
|
+
It is ${input.now}, and they know that.
|
|
39
|
+
They are talking to ${input.talkingTo}. They want: ${input.wants}
|
|
40
|
+
Patience left: about ${Math.max(0, Math.round(input.secondsLeft))} seconds.
|
|
41
|
+
|
|
42
|
+
THE CONVERSATION SO FAR
|
|
43
|
+
${first ? 'Nothing yet. They are about to speak for the first time.' : input.exchanges.map((e, i) => `${i + 1}. They said: "${e.said}"\n They heard: ${e.heard ? `"${e.heard}"` : '(nothing came back)'}`).join('\n')}
|
|
44
|
+
|
|
45
|
+
${first ? 'What is the first thing this person says? The action is "say".' : 'What does this person do next?'}`;
|
|
46
|
+
return {
|
|
47
|
+
role: 'persona',
|
|
48
|
+
system: SYSTEM,
|
|
49
|
+
prompt,
|
|
50
|
+
schema: {
|
|
51
|
+
type: 'object',
|
|
52
|
+
properties: {
|
|
53
|
+
cites: { type: 'string' },
|
|
54
|
+
reason: { type: 'string' },
|
|
55
|
+
action: { type: 'string', enum: ['say', 'done', 'leave'] },
|
|
56
|
+
text: { type: 'string' },
|
|
57
|
+
},
|
|
58
|
+
required: ['cites', 'reason', 'action'],
|
|
59
|
+
},
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
/** Why a proposed action cannot be used, or null if it can. */
|
|
63
|
+
export function problemWith(action, input) {
|
|
64
|
+
const first = input.exchanges.length === 0;
|
|
65
|
+
if (first) {
|
|
66
|
+
if (action.action !== 'say')
|
|
67
|
+
return 'They have not spoken yet. The first action is "say".';
|
|
68
|
+
return action.text?.trim() ? null : 'A "say" action needs the words they say.';
|
|
69
|
+
}
|
|
70
|
+
if (action.action === 'leave')
|
|
71
|
+
return null;
|
|
72
|
+
if (action.action === 'say' && !action.text?.trim())
|
|
73
|
+
return 'A "say" action needs the words they say.';
|
|
74
|
+
const lastHeard = input.exchanges.at(-1).heard;
|
|
75
|
+
// Nothing came back: there is nothing to quote. They may try once more, but cannot claim it worked.
|
|
76
|
+
if (!lastHeard.trim())
|
|
77
|
+
return action.action === 'done' ? 'They heard nothing back, so they cannot believe it worked.' : null;
|
|
78
|
+
if (!citeIsOnScreen(action.cites, [{ text: lastHeard }], [])) {
|
|
79
|
+
return `"cites" must be an exact quote of words they just heard. "${action.cites.slice(0, 80)}" was not said.`;
|
|
80
|
+
}
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Ask for the next action. An unusable one is sent back once. A second
|
|
85
|
+
* failure counts as leaving, except on the first line, where it is an error:
|
|
86
|
+
* a conversation nobody started measures nothing.
|
|
87
|
+
*/
|
|
88
|
+
export async function decideTalker(llm, input) {
|
|
89
|
+
const calls = [];
|
|
90
|
+
let request = buildTalkerRequest(input);
|
|
91
|
+
let problem = '';
|
|
92
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
93
|
+
const response = await llm.complete(request);
|
|
94
|
+
calls.push({ request, response });
|
|
95
|
+
const reply = response.structured;
|
|
96
|
+
if (reply && typeof reply.action === 'string' && ['say', 'done', 'leave'].includes(reply.action)) {
|
|
97
|
+
const action = { cites: '', reason: '', ...reply };
|
|
98
|
+
problem = problemWith(action, input) ?? '';
|
|
99
|
+
if (!problem)
|
|
100
|
+
return { action, calls, forcedLeave: false };
|
|
101
|
+
}
|
|
102
|
+
else {
|
|
103
|
+
problem = 'The reply had no valid action.';
|
|
104
|
+
}
|
|
105
|
+
request = { ...request, prompt: `${request.prompt}\n\nYOUR LAST ANSWER WAS NOT USABLE\n${problem}\nAnswer again.` };
|
|
106
|
+
}
|
|
107
|
+
if (input.exchanges.length === 0)
|
|
108
|
+
throw new Error(`The persona model gave no opening line. ${problem}`);
|
|
109
|
+
return { action: { action: 'leave', cites: '', reason: `No usable next line grounded in what was heard. ${problem}` }, calls, forcedLeave: true };
|
|
110
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
// Talker mode: one persona, one spoken conversation, one trace.
|
|
2
|
+
//
|
|
3
|
+
// This is the brain-only layer. The person's words reach the product as
|
|
4
|
+
// clean text, so nothing here tests hearing: no microphone, no room, no
|
|
5
|
+
// speech recognition. It tests what the product does once it has understood
|
|
6
|
+
// the words.
|
|
7
|
+
//
|
|
8
|
+
// The engine, not the persona model, decides whether the person got what
|
|
9
|
+
// they came for: from what the product did (its events) and, for a prose
|
|
10
|
+
// outcome, from a judge that reads the transcript.
|
|
11
|
+
import { createHash } from 'node:crypto';
|
|
12
|
+
import { readFile } from 'node:fs/promises';
|
|
13
|
+
import { judgeConversation } from "../../judge/conversation.js";
|
|
14
|
+
import { countWords, UNCALIBRATED_DIALS } from "../reader/skim.js";
|
|
15
|
+
import { deriveSeed } from "../rng.js";
|
|
16
|
+
import { validateTrace } from "../trace.js";
|
|
17
|
+
import { decideTalker } from "./decide.js";
|
|
18
|
+
import { assertLoopback } from "./world.js";
|
|
19
|
+
/** Ordinary conversational pace. Uncalibrated. */
|
|
20
|
+
const SPEAK_WPM = 150;
|
|
21
|
+
const LISTEN_WPM = 160;
|
|
22
|
+
export async function runTalker(options) {
|
|
23
|
+
const { spec, persona, world, llm, seed } = options;
|
|
24
|
+
if (!spec.task || !spec.talker)
|
|
25
|
+
throw new Error('Talker mode needs a conversation spec');
|
|
26
|
+
assertLoopback(options.target);
|
|
27
|
+
const success = spec.task.success;
|
|
28
|
+
const budget = spec.talker.patience_seconds;
|
|
29
|
+
const maxTurns = spec.talker.max_turns ?? 6;
|
|
30
|
+
const dials = { ...(options.dials ?? UNCALIBRATED_DIALS), timeBudgetSeconds: budget };
|
|
31
|
+
const startedAt = new Date().toISOString();
|
|
32
|
+
// Local wall-clock time, for the person and for the judge: "tonight" means something only against a clock.
|
|
33
|
+
const now = new Date().toLocaleString('en-US', { weekday: 'long', year: 'numeric', month: 'long', day: 'numeric', hour: 'numeric', minute: '2-digit' });
|
|
34
|
+
const turns = [];
|
|
35
|
+
const exchanges = [];
|
|
36
|
+
const calls = [];
|
|
37
|
+
const allEvents = [];
|
|
38
|
+
let t = 0;
|
|
39
|
+
let outcome;
|
|
40
|
+
let outcomeNote = '';
|
|
41
|
+
let believedDone = false;
|
|
42
|
+
const toolSeen = () => !success.tool_called || allEvents.some((e) => e.name.includes(success.tool_called));
|
|
43
|
+
const conversation = await world.open({ target: options.target, persona, seed });
|
|
44
|
+
try {
|
|
45
|
+
while (!outcome) {
|
|
46
|
+
const start = t;
|
|
47
|
+
if (t >= budget) {
|
|
48
|
+
turns.push({ index: turns.length, action: 'stop', said: '', cites: '', reason: 'patience ran out', heard: '', events: [], waitSeconds: 0, simulatedSecondsStart: round(budget), simulatedSecondsEnd: round(budget) });
|
|
49
|
+
t = budget;
|
|
50
|
+
outcome = 'time_up';
|
|
51
|
+
outcomeNote = 'ran out of patience';
|
|
52
|
+
break;
|
|
53
|
+
}
|
|
54
|
+
if (exchanges.length >= maxTurns) {
|
|
55
|
+
turns.push({ index: turns.length, action: 'stop', said: '', cites: '', reason: 'turn limit reached', heard: '', events: [], waitSeconds: 0, simulatedSecondsStart: round(t), simulatedSecondsEnd: round(t) });
|
|
56
|
+
outcome = 'gave_up';
|
|
57
|
+
outcomeNote = `still going after ${maxTurns} turns`;
|
|
58
|
+
break;
|
|
59
|
+
}
|
|
60
|
+
const decision = await decideTalker(llm, {
|
|
61
|
+
persona, wants: spec.task.instruction ?? spec.task.goal, talkingTo: spec.talker.talking_to ?? 'a voice assistant', exchanges, secondsLeft: budget - t, now,
|
|
62
|
+
});
|
|
63
|
+
calls.push(...decision.calls);
|
|
64
|
+
const a = decision.action;
|
|
65
|
+
if (a.action === 'done' || a.action === 'leave') {
|
|
66
|
+
turns.push({ index: turns.length, action: a.action, said: '', cites: a.cites, reason: a.reason, heard: '', events: [], waitSeconds: 0, simulatedSecondsStart: round(start), simulatedSecondsEnd: round(start) });
|
|
67
|
+
believedDone = a.action === 'done';
|
|
68
|
+
outcome = a.action === 'done' ? 'thought_done' : 'left'; // settled below, once the success checks have run
|
|
69
|
+
outcomeNote = a.reason;
|
|
70
|
+
break;
|
|
71
|
+
}
|
|
72
|
+
t += (countWords(a.text) / SPEAK_WPM) * 60;
|
|
73
|
+
const heard = await conversation.say(a.text);
|
|
74
|
+
allEvents.push(...heard.events);
|
|
75
|
+
t += heard.waitSeconds + (countWords(heard.reply) / LISTEN_WPM) * 60;
|
|
76
|
+
turns.push({
|
|
77
|
+
index: turns.length, action: 'say', said: a.text, cites: a.cites, reason: a.reason,
|
|
78
|
+
...(heard.heardAs !== undefined ? { heardAs: heard.heardAs } : {}),
|
|
79
|
+
heard: heard.reply, events: heard.events, waitSeconds: round(heard.waitSeconds),
|
|
80
|
+
simulatedSecondsStart: round(start), simulatedSecondsEnd: round(Math.min(t, budget)),
|
|
81
|
+
});
|
|
82
|
+
exchanges.push({ said: a.text, heard: heard.reply });
|
|
83
|
+
// With only a mechanical check, the moment the product does it the task is complete.
|
|
84
|
+
if (success.tool_called && !success.outcome && toolSeen()) {
|
|
85
|
+
outcome = 'completed';
|
|
86
|
+
outcomeNote = `the product ran ${success.tool_called}`;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
await conversation.close();
|
|
92
|
+
}
|
|
93
|
+
// Settle the outcome. Every check that the spec sets has to hold: the
|
|
94
|
+
// product did the thing, and the transcript shows the outcome.
|
|
95
|
+
if (outcome !== 'completed') {
|
|
96
|
+
const how = outcomeNote;
|
|
97
|
+
let met = toolSeen();
|
|
98
|
+
let why = met ? '' : `the product never ran ${success.tool_called}`;
|
|
99
|
+
if (met && success.outcome) {
|
|
100
|
+
const verdict = await judgeConversation(llm, success.outcome, now, turns.filter((x) => x.action === 'say').map((x) => ({ said: x.said, heard: x.heard, events: x.events })));
|
|
101
|
+
calls.push(...verdict.calls);
|
|
102
|
+
met = verdict.met;
|
|
103
|
+
why = verdict.note;
|
|
104
|
+
}
|
|
105
|
+
else if (met && !success.tool_called) {
|
|
106
|
+
met = false; // nothing to check against: the spec loader refuses this, kept as a guard
|
|
107
|
+
}
|
|
108
|
+
if (met) {
|
|
109
|
+
outcome = 'completed';
|
|
110
|
+
outcomeNote = success.outcome ? `the transcript shows the outcome: ${why}` : `the product ran ${success.tool_called}`;
|
|
111
|
+
}
|
|
112
|
+
else if (believedDone) {
|
|
113
|
+
outcome = 'thought_done';
|
|
114
|
+
outcomeNote = `believed it worked, but it did not: ${why}`;
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
outcomeNote = `${how}. Check: ${why}`;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const usd = calls.reduce((sum, c) => sum + c.response.costUsd, 0);
|
|
121
|
+
const done = outcome === 'completed';
|
|
122
|
+
const spoken = turns.filter((x) => x.action === 'say').length;
|
|
123
|
+
const trace = {
|
|
124
|
+
schema: 'sdd-trace/1',
|
|
125
|
+
runId: `${startedAt.replace(/[:.]/g, '-')}_${persona.id}_voice_s${seed}`,
|
|
126
|
+
...(options.batch ? { batch: options.batch } : {}),
|
|
127
|
+
experiment: spec.experiment,
|
|
128
|
+
mode: 'talker',
|
|
129
|
+
startedAt,
|
|
130
|
+
page: { path: `${options.target} (conversation)`, sha256: createHash('sha256').update(await readFile(options.worldFile)).digest('hex'), words: 0 },
|
|
131
|
+
persona,
|
|
132
|
+
personaSet: options.personaSet ?? 'working',
|
|
133
|
+
viewport: { name: 'voice', width: 0, height: 0 },
|
|
134
|
+
seed,
|
|
135
|
+
derivedSeed: deriveSeed(spec.experiment, persona.id, 'voice', seed),
|
|
136
|
+
dials: { ...dials, calibrated: false },
|
|
137
|
+
steps: [],
|
|
138
|
+
talkerTurns: turns,
|
|
139
|
+
recalled: [],
|
|
140
|
+
outcome: outcome,
|
|
141
|
+
simulatedSeconds: round(Math.min(t, budget)),
|
|
142
|
+
// "Where they stopped": how many lines they had spoken when it ended.
|
|
143
|
+
deepestSection: `after ${spoken} ${spoken === 1 ? 'line' : 'lines'}`,
|
|
144
|
+
answers: [{ id: 'task', ask: spec.task.goal, answer: `${outcome}: ${outcomeNote}` }],
|
|
145
|
+
grades: [{ questionId: 'task', pass: done, requiredMet: [done], bonusMet: [], note: done ? 'complete' : outcomeNote }],
|
|
146
|
+
llmCalls: calls,
|
|
147
|
+
cost: {
|
|
148
|
+
usd,
|
|
149
|
+
inputTokens: calls.reduce((s, c) => s + c.response.usage.inputTokens + c.response.usage.cacheReadTokens + c.response.usage.cacheWriteTokens, 0),
|
|
150
|
+
outputTokens: calls.reduce((s, c) => s + c.response.usage.outputTokens, 0),
|
|
151
|
+
},
|
|
152
|
+
sandbox: { origin: options.target, violations: 0 },
|
|
153
|
+
};
|
|
154
|
+
validateTrace(trace, ['task']);
|
|
155
|
+
return trace;
|
|
156
|
+
}
|
|
157
|
+
function round(n) {
|
|
158
|
+
return Math.round(n * 1000) / 1000;
|
|
159
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// The world a talker speaks into. The engine knows nothing about the product
|
|
2
|
+
// on the other end: a work directory supplies an adapter that opens one
|
|
3
|
+
// conversation and passes lines back and forth.
|
|
4
|
+
//
|
|
5
|
+
// An adapter is part of the referee. It decides what counts as "what the
|
|
6
|
+
// product said" and "what the product did", so a revising agent may not edit it.
|
|
7
|
+
//
|
|
8
|
+
// Hard rule 1 holds here too: the target must be on this machine. The engine
|
|
9
|
+
// checks the address. The adapter must talk to nothing else.
|
|
10
|
+
var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
|
|
11
|
+
if (typeof path === "string" && /^\.\.?\//.test(path)) {
|
|
12
|
+
return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
|
|
13
|
+
return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
return path;
|
|
17
|
+
};
|
|
18
|
+
import path from 'node:path';
|
|
19
|
+
import { pathToFileURL } from 'node:url';
|
|
20
|
+
export function assertLoopback(target) {
|
|
21
|
+
const url = new URL(target);
|
|
22
|
+
if (!['localhost', '127.0.0.1', '[::1]'].includes(url.hostname) || !/^(https?|wss?):$/.test(url.protocol)) {
|
|
23
|
+
throw new Error(`A conversation target must be a loopback address, got ${target}`);
|
|
24
|
+
}
|
|
25
|
+
return url;
|
|
26
|
+
}
|
|
27
|
+
/** Loads <workdir>/<adapter>. The module's default export is the World. */
|
|
28
|
+
export async function loadWorld(repoRoot, adapter) {
|
|
29
|
+
const file = path.resolve(repoRoot, adapter);
|
|
30
|
+
if (!file.startsWith(path.resolve(repoRoot) + path.sep))
|
|
31
|
+
throw new Error(`The adapter must live inside the work directory: ${adapter}`);
|
|
32
|
+
const module = (await import(__rewriteRelativeImportExtension(pathToFileURL(file).href)));
|
|
33
|
+
if (!module.default || typeof module.default.open !== 'function')
|
|
34
|
+
throw new Error(`${adapter}: the default export must have an open() function`);
|
|
35
|
+
return { world: module.default, file };
|
|
36
|
+
}
|