sdd-engine 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +178 -2
  3. package/dist/engine/cli.js +95 -0
  4. package/dist/engine/simulate.js +214 -0
  5. package/dist/judge/conversation.js +45 -0
  6. package/dist/judge/judge.js +122 -0
  7. package/dist/judge/outcome.js +42 -0
  8. package/dist/judge/regrade.js +74 -0
  9. package/dist/pipeline/agent.js +78 -0
  10. package/dist/pipeline/agentstage.js +88 -0
  11. package/dist/pipeline/approve.js +68 -0
  12. package/dist/pipeline/cli.js +261 -0
  13. package/dist/pipeline/derive.js +287 -0
  14. package/dist/pipeline/designloop.js +385 -0
  15. package/dist/pipeline/facts.js +108 -0
  16. package/dist/pipeline/mock.js +253 -0
  17. package/dist/pipeline/project.js +79 -0
  18. package/dist/pipeline/render.js +95 -0
  19. package/dist/pipeline/spec.js +272 -0
  20. package/dist/pipeline/try.js +162 -0
  21. package/dist/runner/batch-lib.js +157 -0
  22. package/dist/runner/batch.js +56 -0
  23. package/dist/runner/cli.js +61 -0
  24. package/dist/runner/llm/claude-code.js +123 -0
  25. package/dist/runner/llm/index.js +12 -0
  26. package/dist/runner/llm/types.js +17 -0
  27. package/dist/runner/reader/answer.js +76 -0
  28. package/dist/runner/reader/extract.js +106 -0
  29. package/dist/runner/reader/run.js +111 -0
  30. package/dist/runner/reader/skim.js +115 -0
  31. package/dist/runner/rng.js +22 -0
  32. package/dist/runner/sandbox/cli.js +60 -0
  33. package/dist/runner/sandbox/sandbox.js +144 -0
  34. package/dist/runner/sandbox/server.js +64 -0
  35. package/dist/runner/spec.js +84 -0
  36. package/dist/runner/talker/decide.js +110 -0
  37. package/dist/runner/talker/run.js +159 -0
  38. package/dist/runner/talker/world.js +36 -0
  39. package/dist/runner/trace.js +130 -0
  40. package/dist/runner/visitor/decide.js +159 -0
  41. package/dist/runner/visitor/observe.js +102 -0
  42. package/dist/runner/visitor/run.js +296 -0
  43. package/dist/scorer/compare.js +119 -0
  44. package/dist/scorer/score.js +201 -0
  45. package/dist/scorer/spot-check-compare.js +41 -0
  46. package/dist/scorer/spot-check-view.js +45 -0
  47. package/dist/scorer/stats.js +45 -0
  48. package/dist/scorer/summary.js +62 -0
  49. package/package.json +49 -4
@@ -0,0 +1,272 @@
1
+ // The pipeline spec: what a result means, made measurable.
2
+ //
3
+ // Derived from one sentence by the deriver, approved by the owner, and then
4
+ // fixed. It names who the result is for, the stages a person goes through to
5
+ // get there, what counts as success at each, the share who must succeed, and
6
+ // the assumptions the simulation runs under.
7
+ // ---- Validation ----
8
+ const SLUG = /^[a-z][a-z0-9-]{1,39}$/;
9
+ export class Issues {
10
+ list = [];
11
+ add(where, message) {
12
+ this.list.push(`${where}: ${message}`);
13
+ }
14
+ throwIfAny(what) {
15
+ if (this.list.length > 0)
16
+ throw new Error(`${what} is not usable:\n${this.list.map((i) => ` - ${i}`).join('\n')}`);
17
+ }
18
+ }
19
+ export const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
20
+ export function text(issues, where, value, min, max) {
21
+ if (typeof value !== 'string') {
22
+ issues.add(where, 'must be text');
23
+ return '';
24
+ }
25
+ const t = value.trim();
26
+ if (t.length < min || t.length > max)
27
+ issues.add(where, `must be ${min} to ${max} characters`);
28
+ return t;
29
+ }
30
+ function slug(issues, where, value) {
31
+ if (typeof value !== 'string' || !SLUG.test(value)) {
32
+ issues.add(where, 'must be lowercase letters, digits and dashes, starting with a letter');
33
+ return '';
34
+ }
35
+ return value;
36
+ }
37
+ function num(issues, where, value, min, max, whole = false) {
38
+ if (typeof value !== 'number' || !Number.isFinite(value) || value < min || value > max || (whole && !Number.isInteger(value))) {
39
+ issues.add(where, `must be ${whole ? 'a whole number' : 'a number'} from ${min} to ${max}`);
40
+ return min;
41
+ }
42
+ return value;
43
+ }
44
+ export function list(issues, where, value, min, max, each) {
45
+ if (!Array.isArray(value)) {
46
+ issues.add(where, 'must be a list');
47
+ return [];
48
+ }
49
+ if (value.length < min || value.length > max)
50
+ issues.add(where, `must have ${min} to ${max} items`);
51
+ return value.map((item, i) => each(item, `${where}[${i}]`));
52
+ }
53
+ function viewports(issues, where, value) {
54
+ const out = list(issues, where, value, 1, 2, (v, w) => {
55
+ if (v !== 'mobile' && v !== 'desktop')
56
+ issues.add(w, 'must be "mobile" or "desktop"');
57
+ return v;
58
+ });
59
+ return [...new Set(out)];
60
+ }
61
+ function unique(issues, where, ids) {
62
+ if (new Set(ids).size !== ids.length)
63
+ issues.add(where, 'ids must be unique');
64
+ }
65
+ function parseStage(issues, item, where) {
66
+ const s = isObject(item) ? item : {};
67
+ if (!isObject(item))
68
+ issues.add(where, 'must be an object');
69
+ const id = slug(issues, `${where}.id`, s.id);
70
+ const goal = text(issues, `${where}.goal`, s.goal, 10, 400);
71
+ if (s.kind === 'read') {
72
+ return {
73
+ id, kind: 'read', goal,
74
+ entryFrom: text(issues, `${where}.entryFrom`, s.entryFrom, 3, 200),
75
+ viewports: viewports(issues, `${where}.viewports`, s.viewports),
76
+ timeBudgetSeconds: num(issues, `${where}.timeBudgetSeconds`, s.timeBudgetSeconds, 15, 180, true),
77
+ questions: list(issues, `${where}.questions`, s.questions, 2, 4, (q, w) => {
78
+ const o = isObject(q) ? q : {};
79
+ if (!isObject(q))
80
+ issues.add(w, 'must be an object');
81
+ return {
82
+ id: slug(issues, `${w}.id`, o.id),
83
+ ask: text(issues, `${w}.ask`, o.ask, 10, 300),
84
+ required: list(issues, `${w}.required`, o.required, 1, 3, (v, x) => text(issues, x, v, 5, 300)),
85
+ threshold: num(issues, `${w}.threshold`, o.threshold, 0.05, 0.95),
86
+ reason: text(issues, `${w}.reason`, o.reason, 10, 600),
87
+ };
88
+ }),
89
+ };
90
+ }
91
+ if (s.kind === 'decide') {
92
+ const success = isObject(s.success) ? s.success : {};
93
+ const leavesTo = list(issues, `${where}.success.leavesTo`, success.leavesTo ?? [], 0, 6, (v, w) => text(issues, w, v, 3, 200));
94
+ const clicks = list(issues, `${where}.success.clicks`, success.clicks ?? [], 0, 6, (v, w) => text(issues, w, v, 2, 80));
95
+ if (leavesTo.length + clicks.length === 0)
96
+ issues.add(`${where}.success`, 'needs at least one leavesTo or clicks entry');
97
+ return {
98
+ id, kind: 'decide', goal,
99
+ entryFrom: text(issues, `${where}.entryFrom`, s.entryFrom, 3, 200),
100
+ viewports: viewports(issues, `${where}.viewports`, s.viewports),
101
+ success: { leavesTo, clicks },
102
+ patienceSeconds: num(issues, `${where}.patienceSeconds`, s.patienceSeconds, 20, 300, true),
103
+ threshold: num(issues, `${where}.threshold`, s.threshold, 0.05, 0.95),
104
+ reason: text(issues, `${where}.reason`, s.reason, 10, 600),
105
+ };
106
+ }
107
+ if (s.kind === 'try') {
108
+ const machine = isObject(s.machine) ? s.machine : {};
109
+ const success = isObject(s.success) ? s.success : {};
110
+ return {
111
+ id, kind: 'try', goal,
112
+ machine: {
113
+ image: text(issues, `${where}.machine.image`, machine.image, 3, 120),
114
+ preinstalled: list(issues, `${where}.machine.preinstalled`, machine.preinstalled, 0, 12, (v, w) => text(issues, w, v, 2, 120)),
115
+ notPresent: list(issues, `${where}.machine.notPresent`, machine.notPresent, 0, 12, (v, w) => text(issues, w, v, 2, 160)),
116
+ },
117
+ given: text(issues, `${where}.given`, s.given, 10, 400),
118
+ success: {
119
+ observable: text(issues, `${where}.success.observable`, success.observable, 10, 400),
120
+ check: text(issues, `${where}.success.check`, success.check, 10, 400),
121
+ },
122
+ timeLimitSeconds: num(issues, `${where}.timeLimitSeconds`, s.timeLimitSeconds, 30, 3600, true),
123
+ threshold: num(issues, `${where}.threshold`, s.threshold, 0.05, 0.95),
124
+ reason: text(issues, `${where}.reason`, s.reason, 10, 600),
125
+ };
126
+ }
127
+ if (s.kind === 'agent') {
128
+ const requests = list(issues, `${where}.requests`, s.requests, 4, 10, (r, w) => {
129
+ const o = isObject(r) ? r : {};
130
+ if (!isObject(r))
131
+ issues.add(w, 'must be an object');
132
+ if (typeof o.fits !== 'boolean')
133
+ issues.add(`${w}.fits`, 'must be true or false');
134
+ if (typeof o.holdout !== 'boolean')
135
+ issues.add(`${w}.holdout`, 'must be true or false');
136
+ return { id: slug(issues, `${w}.id`, o.id), ask: text(issues, `${w}.ask`, o.ask, 15, 400), fits: o.fits === true, holdout: o.holdout === true };
137
+ });
138
+ unique(issues, `${where}.requests`, requests.map((r) => r.id));
139
+ const seen = requests.filter((r) => !r.holdout);
140
+ const held = requests.filter((r) => r.holdout);
141
+ if (seen.filter((r) => r.fits).length < 2)
142
+ issues.add(`${where}.requests`, 'needs at least 2 requests the product fits that are not held back');
143
+ if (!seen.some((r) => !r.fits))
144
+ issues.add(`${where}.requests`, 'needs at least 1 request the product does not fit that is not held back');
145
+ if (!held.some((r) => r.fits))
146
+ issues.add(`${where}.requests`, 'needs at least 1 held-back request the product fits');
147
+ return {
148
+ id, kind: 'agent', goal, requests,
149
+ required: list(issues, `${where}.required`, s.required, 1, 4, (v, w) => text(issues, w, v, 5, 300)),
150
+ threshold: num(issues, `${where}.threshold`, s.threshold, 0.05, 0.95),
151
+ reason: text(issues, `${where}.reason`, s.reason, 10, 600),
152
+ };
153
+ }
154
+ issues.add(`${where}.kind`, 'must be "read", "decide", "try" or "agent"');
155
+ return { id, kind: 'read', goal, entryFrom: '', viewports: [], timeBudgetSeconds: 15, questions: [] };
156
+ }
157
+ /** Checks a derived spec. Throws one error that lists every problem. */
158
+ export function parsePipelineSpec(input, result) {
159
+ const issues = new Issues();
160
+ if (!isObject(input))
161
+ throw new Error('The spec must be a JSON object.');
162
+ const audience = isObject(input.audience) ? input.audience : {};
163
+ if (!isObject(input.audience))
164
+ issues.add('audience', 'must be an object');
165
+ const personas = list(issues, 'personas', input.personas, 4, 8, (item, where) => {
166
+ const p = isObject(item) ? item : {};
167
+ if (!isObject(item))
168
+ issues.add(where, 'must be an object');
169
+ if (typeof p.holdout !== 'boolean')
170
+ issues.add(`${where}.holdout`, 'must be true or false');
171
+ return {
172
+ id: slug(issues, `${where}.id`, p.id),
173
+ archetype: text(issues, `${where}.archetype`, p.archetype, 5, 200),
174
+ role: text(issues, `${where}.role`, p.role, 2, 200),
175
+ experience: text(issues, `${where}.experience`, p.experience, 2, 200),
176
+ alreadyUses: list(issues, `${where}.alreadyUses`, p.alreadyUses, 1, 5, (v, w) => text(issues, w, v, 1, 200)),
177
+ arrivalSource: text(issues, `${where}.arrivalSource`, p.arrivalSource, 2, 100),
178
+ arrivalSaw: text(issues, `${where}.arrivalSaw`, p.arrivalSaw, 2, 300),
179
+ needs: list(issues, `${where}.needs`, p.needs, 2, 3, (v, w) => text(issues, w, v, 3, 300)),
180
+ objections: list(issues, `${where}.objections`, p.objections, 2, 3, (v, w) => text(issues, w, v, 3, 300)),
181
+ readingWpm: num(issues, `${where}.readingWpm`, p.readingWpm, 150, 400, true),
182
+ skimRate: num(issues, `${where}.skimRate`, p.skimRate, 0.2, 0.85),
183
+ holdout: p.holdout === true,
184
+ };
185
+ });
186
+ const stages = list(issues, 'stages', input.stages, 1, 6, (item, where) => parseStage(issues, item, where));
187
+ const stageIds = stages.map((s) => s.id);
188
+ const assumptions = list(issues, 'assumptions', input.assumptions, 1, 12, (item, where) => {
189
+ const a = isObject(item) ? item : {};
190
+ if (!isObject(item))
191
+ issues.add(where, 'must be an object');
192
+ if (a.source !== 'measured' && a.source !== 'assumed')
193
+ issues.add(`${where}.source`, 'must be "measured" or "assumed"');
194
+ if (typeof a.value !== 'number' && typeof a.value !== 'string')
195
+ issues.add(`${where}.value`, 'must be a number or text');
196
+ const sweep = list(issues, `${where}.sweep`, a.sweep, 2, 8, (v, w) => {
197
+ if (typeof v !== 'number' && typeof v !== 'string')
198
+ issues.add(w, 'must be a number or text');
199
+ return v;
200
+ });
201
+ const affects = list(issues, `${where}.affects`, a.affects, 1, 6, (v, w) => {
202
+ if (typeof v !== 'string' || !stageIds.includes(v))
203
+ issues.add(w, `must be the id of a stage (${stageIds.join(', ')})`);
204
+ return String(v);
205
+ });
206
+ return {
207
+ id: slug(issues, `${where}.id`, a.id),
208
+ statement: text(issues, `${where}.statement`, a.statement, 10, 300),
209
+ value: a.value,
210
+ unit: typeof a.unit === 'string' && a.unit.trim() ? a.unit.trim() : undefined,
211
+ source: a.source === 'measured' ? 'measured' : 'assumed',
212
+ // "Measured" has to say how and what was read. "Assumed" has to say why.
213
+ how: text(issues, `${where}.how`, a.how, 15, 500),
214
+ sweep,
215
+ affects,
216
+ };
217
+ });
218
+ const spec = {
219
+ schema: 'sdd-pipeline-spec/1',
220
+ result: text(issues, 'result', input.result, 10, 500),
221
+ audience: {
222
+ who: text(issues, 'audience.who', audience.who, 20, 600),
223
+ inferredFrom: list(issues, 'audience.inferredFrom', audience.inferredFrom, 1, 8, (v, w) => text(issues, w, v, 5, 300)),
224
+ },
225
+ personas,
226
+ stages,
227
+ assumptions,
228
+ openQuestions: list(issues, 'openQuestions', input.openQuestions ?? [], 0, 8, (v, w) => text(issues, w, v, 10, 500)),
229
+ };
230
+ // The owner's sentence is not the deriver's to reword.
231
+ if (result !== undefined && spec.result !== result.trim())
232
+ issues.add('result', 'must be the owner\'s sentence, word for word');
233
+ const working = personas.filter((p) => !p.holdout).length;
234
+ if (working < 3)
235
+ issues.add('personas', 'at least 3 must have holdout: false');
236
+ if (personas.length - working < 1)
237
+ issues.add('personas', 'at least 1 must have holdout: true');
238
+ unique(issues, 'personas', personas.map((p) => p.id));
239
+ unique(issues, 'stages', stageIds);
240
+ unique(issues, 'assumptions', assumptions.map((a) => a.id));
241
+ for (const s of stages)
242
+ if (s.kind === 'read')
243
+ unique(issues, `stage ${s.id} questions`, s.questions.map((q) => q.id));
244
+ issues.throwIfAny('The spec');
245
+ return spec;
246
+ }
247
+ /** Every piece of text in a value, with where it sits. For checks that apply to all copy. */
248
+ export function allText(value, where = '') {
249
+ if (typeof value === 'string')
250
+ return [{ where, text: value }];
251
+ if (Array.isArray(value))
252
+ return value.flatMap((v, i) => allText(v, `${where}[${i}]`));
253
+ if (isObject(value))
254
+ return Object.entries(value).flatMap(([k, v]) => allText(v, where ? `${where}.${k}` : k));
255
+ return [];
256
+ }
257
+ /** The house rule: no em dashes in any copy or docs. */
258
+ export function emDashProblems(value) {
259
+ return allText(value).filter((t) => t.text.includes('—')).map((t) => `${t.where}: contains an em dash`);
260
+ }
261
+ /** A persona in the engine's file format. */
262
+ export function enginePersona(p) {
263
+ return {
264
+ id: p.id,
265
+ archetype: p.archetype,
266
+ profile: { role: p.role, experience: p.experience, tech_comfort: 'not stated', already_uses: p.alreadyUses },
267
+ arrival: { source: p.arrivalSource, saw: p.arrivalSaw },
268
+ needs: p.needs,
269
+ objections: p.objections,
270
+ dials: { reading_wpm: p.readingWpm, skim_rate: p.skimRate },
271
+ };
272
+ }
@@ -0,0 +1,162 @@
1
+ // The "try" stage: a developer at a terminal on a clean machine, handed only
2
+ // what the page says.
3
+ //
4
+ // The machine is a fresh container. The developer is a model with no tools:
5
+ // it reads the page's text and the terminal so far, and names the next
6
+ // command. The pipeline runs that command in the container and captures what
7
+ // really came back. Whether it worked is judged from that captured output
8
+ // against the spec's check, never from anything the developer model claims.
9
+ import { spawn } from 'node:child_process';
10
+ function run(cmd, args, timeoutMs) {
11
+ return new Promise((resolve) => {
12
+ const child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'] });
13
+ let out = '';
14
+ const timer = setTimeout(() => child.kill('SIGKILL'), timeoutMs);
15
+ child.stdout.on('data', (d) => (out += d));
16
+ child.stderr.on('data', (d) => (out += d));
17
+ child.on('error', (e) => { clearTimeout(timer); resolve({ code: null, out: String(e) }); });
18
+ child.on('close', (code) => { clearTimeout(timer); resolve({ code, out }); });
19
+ });
20
+ }
21
+ /** The image name from the spec's description of the machine: its first word. */
22
+ export const imageOf = (stage) => stage.machine.image.trim().split(/[\s(,]/)[0];
23
+ /**
24
+ * A fresh container, exactly as the image ships: its ordinary non-root user,
25
+ * no terminal attached, no environment beyond the image's own. Nothing is set
26
+ * up for the developer, so a global npm install fails here as it does for real.
27
+ */
28
+ export async function startContainer(stage, name) {
29
+ const image = imageOf(stage);
30
+ const started = await run('docker', [
31
+ 'run', '-d', '--rm', '--name', name, '--user', 'node', '-w', '/home/node',
32
+ image, 'sleep', '3600',
33
+ ], 120_000);
34
+ if (started.code !== 0)
35
+ throw new Error(`Could not start a clean ${image} machine: ${started.out.slice(0, 300)}`);
36
+ return {
37
+ async exec(command, timeoutSeconds) {
38
+ const t0 = Date.now();
39
+ // No terminal and no stdin, as the spec's machine says. timeout(1) ends a command that hangs.
40
+ const r = await run('docker', ['exec', name, 'timeout', '-s', 'KILL', String(timeoutSeconds), 'bash', '-lc', command], (timeoutSeconds + 15) * 1000);
41
+ return { exitCode: r.code, output: r.out.slice(-6000), seconds: (Date.now() - t0) / 1000 };
42
+ },
43
+ async destroy() {
44
+ await run('docker', ['rm', '-f', name], 60_000);
45
+ },
46
+ };
47
+ }
48
+ const DEVELOPER_SYSTEM = `You predict what a developer does at a terminal on a machine they have just been given.
49
+
50
+ They have read one web page and know nothing else about the product on it. They were asked to try it. They follow what the page says, in the order it says it, the way a competent developer in a hurry would: they copy commands as written, adapt the obvious placeholder, read errors, and try the next thing the page suggests.
51
+ - They do not know anything the page does not say. Do not use outside knowledge of this product.
52
+ - They can make a folder, change into it, and run commands. Each command you give runs in a fresh shell in their home folder, so chain what belongs together with && (for example: mkdir -p todo && cd todo && <command>).
53
+ - There is no terminal attached: nothing interactive can be answered.
54
+ - They stop when the product has clearly run and told them something they cannot get past from the page alone, or when the page gives them nothing more to try.
55
+ Give one command, or stop.`;
56
+ const STEP_SCHEMA = {
57
+ type: 'object',
58
+ properties: {
59
+ action: { type: 'string', enum: ['run', 'stop'] },
60
+ command: { type: 'string', description: 'One shell command line. Empty when stopping.' },
61
+ why: { type: 'string', description: 'One line: what on the page led to this.' },
62
+ },
63
+ required: ['action', 'command', 'why'],
64
+ additionalProperties: false,
65
+ };
66
+ const CHECK_SYSTEM = `You decide whether something observable happened, from a terminal transcript.
67
+
68
+ You get the thing that must be observed, how it is to be seen, and the real transcript: each command that was run, its exit code, and what it printed. The transcript was captured by the test, not written by the person at the terminal.
69
+ - Judge only from command output and exit codes. Text that the person typed themselves, such as an echo of the expected message, does not count.
70
+ - If the check names specific output and an exit code, both have to be there, from the same command.
71
+ Answer whether it was met, with one line pointing at the command and output that shows it.`;
72
+ /** What the developer is asked to do: the quoted sentence in the spec's "given", or the stage goal. */
73
+ export function taskSentence(stage) {
74
+ // An opening quote follows a space. That skips the apostrophe in words like "site's".
75
+ return stage.given.match(/(?:^|\s)'([^']{10,})'/)?.[1] ?? stage.goal;
76
+ }
77
+ async function checkMet(llm, stage, steps) {
78
+ if (steps.length === 0)
79
+ return { met: false, note: 'No command was run.', costUsd: 0 };
80
+ const response = await llm.complete({
81
+ role: 'judge',
82
+ system: CHECK_SYSTEM,
83
+ prompt: [
84
+ `What must be observed: ${stage.success.observable}`,
85
+ `How it is seen: ${stage.success.check}`,
86
+ '',
87
+ '# Transcript',
88
+ ...steps.map((s, i) => `\n## Command ${i + 1} (exit code ${s.exitCode}, ${s.seconds.toFixed(0)} s)\n$ ${s.command}\n${s.output || '(no output)'}`),
89
+ ].join('\n'),
90
+ schema: { type: 'object', properties: { met: { type: 'boolean' }, note: { type: 'string' } }, required: ['met', 'note'], additionalProperties: false },
91
+ });
92
+ const out = response.structured;
93
+ return { met: out?.met === true, note: out?.note ?? '', costUsd: response.costUsd };
94
+ }
95
+ export async function tryOnce(options) {
96
+ const { stage, llm, machine } = options;
97
+ const maxSteps = options.maxSteps ?? 10;
98
+ const steps = [];
99
+ let costUsd = 0;
100
+ let ended = 'step limit';
101
+ const t0 = Date.now();
102
+ const elapsed = () => (Date.now() - t0) / 1000;
103
+ try {
104
+ while (steps.length < maxSteps) {
105
+ if (elapsed() >= stage.timeLimitSeconds) {
106
+ ended = 'time limit';
107
+ break;
108
+ }
109
+ const response = await llm.complete({
110
+ role: 'judge',
111
+ system: DEVELOPER_SYSTEM,
112
+ prompt: [
113
+ '# The page they read', '', options.pageText, '',
114
+ `# What they were asked`, '', taskSentence(stage), '',
115
+ '# The terminal so far', '',
116
+ steps.length === 0 ? '(nothing run yet)' : steps.map((s) => `$ ${s.command}\n${s.output.slice(-1500) || '(no output)'}\n[exit code ${s.exitCode}]`).join('\n\n'),
117
+ '', 'What do they do next?',
118
+ ].join('\n'),
119
+ schema: STEP_SCHEMA,
120
+ });
121
+ costUsd += response.costUsd;
122
+ const step = response.structured;
123
+ if (!step || step.action !== 'run' || !step.command?.trim()) {
124
+ ended = 'stopped';
125
+ break;
126
+ }
127
+ const left = Math.max(5, Math.floor(stage.timeLimitSeconds - elapsed()));
128
+ const result = await machine.exec(step.command, Math.min(options.commandTimeoutSeconds ?? 150, left));
129
+ steps.push({ command: step.command, why: step.why ?? '', ...result });
130
+ }
131
+ const verdict = await checkMet(llm, stage, steps);
132
+ costUsd += verdict.costUsd;
133
+ return { n: options.n, met: verdict.met, ended: verdict.met ? 'check met' : ended, note: verdict.note, steps, seconds: elapsed(), costUsd };
134
+ }
135
+ catch (err) {
136
+ return { n: options.n, met: false, ended: 'error', note: err.message.slice(0, 300), steps, seconds: elapsed(), costUsd };
137
+ }
138
+ }
139
+ /** Several developers, each on a machine of their own. */
140
+ export async function tryStage(stage, pageText, llm, count, start, concurrency = 3) {
141
+ const runs = [];
142
+ let next = 1;
143
+ async function worker() {
144
+ while (next <= count) {
145
+ const n = next++;
146
+ let machine;
147
+ try {
148
+ machine = await start(n);
149
+ runs.push(await tryOnce({ stage, pageText, llm, machine, n }));
150
+ }
151
+ catch (err) {
152
+ runs.push({ n, met: false, ended: 'error', note: err.message.slice(0, 300), steps: [], seconds: 0, costUsd: 0 });
153
+ }
154
+ finally {
155
+ await machine?.destroy();
156
+ }
157
+ }
158
+ }
159
+ await Promise.all(Array.from({ length: Math.min(concurrency, count) }, worker));
160
+ runs.sort((a, b) => a.n - b.n);
161
+ return { runs, passed: runs.filter((r) => r.met).length, n: runs.length, costUsd: runs.reduce((s, r) => s + r.costUsd, 0) };
162
+ }
@@ -0,0 +1,157 @@
1
+ // Runs a batch of graded reader runs. Used by the batch command line and by
2
+ // the automated loop.
3
+ //
4
+ // Safe to call again with the same batch id: runs that already have a trace
5
+ // are skipped. Stops starting new runs when the budget is spent (hard rule 6).
6
+ import { readdir, readFile } from 'node:fs/promises';
7
+ import path from 'node:path';
8
+ import { gradeTrace } from "../judge/judge.js";
9
+ import { runReader } from "./reader/run.js";
10
+ import { runVisitor } from "./visitor/run.js";
11
+ import { runTalker } from "./talker/run.js";
12
+ import { assertLoopback, loadWorld } from "./talker/world.js";
13
+ import { startSandbox } from "./sandbox/sandbox.js";
14
+ import { loadPersona } from "./spec.js";
15
+ import { validateTrace, writeTrace } from "./trace.js";
16
+ /** Collapse refused URLs to their origins, most frequent first. */
17
+ export function blockedOrigins(urls) {
18
+ const counts = new Map();
19
+ for (const url of urls) {
20
+ let origin = url;
21
+ try {
22
+ const parsed = new URL(url).origin;
23
+ if (parsed !== 'null')
24
+ origin = parsed;
25
+ }
26
+ catch { /* keep the raw value, such as a CONNECT host:port */ }
27
+ counts.set(origin, (counts.get(origin) ?? 0) + 1);
28
+ }
29
+ return [...counts]
30
+ .map(([origin, count]) => ({ origin, count }))
31
+ .sort((a, b) => b.count - a.count || a.origin.localeCompare(b.origin));
32
+ }
33
+ export async function loadPersonaSet(repoRoot, dir) {
34
+ const files = (await readdir(path.resolve(repoRoot, dir))).filter((f) => f.endsWith('.yaml')).sort();
35
+ return Promise.all(files.map((f) => loadPersona(path.resolve(repoRoot, dir, f))));
36
+ }
37
+ export async function loadBatchTraces(repoRoot, experiment, batch) {
38
+ const dir = path.join(repoRoot, 'traces', experiment);
39
+ const traces = [];
40
+ for (const file of (await readdir(dir).catch(() => [])).sort()) {
41
+ if (!file.endsWith('.json'))
42
+ continue;
43
+ const trace = JSON.parse(await readFile(path.join(dir, file), 'utf8'));
44
+ if (trace.batch === batch)
45
+ traces.push(trace);
46
+ }
47
+ return traces;
48
+ }
49
+ export async function runBatch(options) {
50
+ const { repoRoot, spec, batch, llm } = options;
51
+ const log = options.log ?? console.log;
52
+ const pagePath = options.targetOrigin ? undefined : (options.pagePath ?? spec.page);
53
+ if (!options.targetOrigin && !pagePath)
54
+ throw new Error('The spec names no page and no target was given');
55
+ const label = options.targetOrigin ? `${options.targetOrigin}${options.entryPath ?? '/'}` : pagePath;
56
+ const questionIds = spec.questions.map((q) => q.id);
57
+ const tracesDir = path.join(repoRoot, 'traces');
58
+ let personas = [];
59
+ for (const set of options.sets) {
60
+ const loaded = await loadPersonaSet(repoRoot, set === 'working' ? spec.personas.working : spec.personas.holdout);
61
+ personas.push(...loaded.map((persona) => ({ set, persona })));
62
+ }
63
+ if (options.personaIds)
64
+ personas = personas.filter((p) => options.personaIds.includes(p.persona.id));
65
+ const key = (personaId, viewport, seed) => `${personaId}|${viewport}|${seed}`;
66
+ const existing = (await loadBatchTraces(repoRoot, spec.experiment, batch)).filter((t) => options.sets.includes(t.personaSet));
67
+ const done = new Set(existing.map((t) => key(t.persona.id, t.viewport.name, t.seed)));
68
+ const tasks = [];
69
+ for (const { set, persona } of personas) {
70
+ for (const viewport of options.viewports) {
71
+ for (let seed = 1; seed <= options.runs; seed++) {
72
+ if (!done.has(key(persona.id, viewport, seed)))
73
+ tasks.push({ set, persona, viewport, seed });
74
+ }
75
+ }
76
+ }
77
+ log(`Batch "${batch}" on ${label}: ${personas.length} personas x ${options.viewports.length} viewports x ${options.runs} seeds`);
78
+ log(` ${done.size} already done, ${tasks.length} to run, budget $${options.budgetUsd.toFixed(2)}`);
79
+ const traces = [...existing];
80
+ let spent = 0, completed = 0, failed = 0, budgetHit = false, next = 0;
81
+ if (tasks.length === 0)
82
+ return { traces, newRuns: 0, failed: 0, spentUsd: 0, budgetHit: false, blocked: [] };
83
+ // A conversation has no browser, so no sandbox: the adapter talks to the
84
+ // target, and the target must be on this machine.
85
+ const talking = spec.kind === 'conversation';
86
+ if (talking && !options.targetOrigin)
87
+ throw new Error('A conversation spec needs a running target');
88
+ if (talking)
89
+ assertLoopback(options.targetOrigin);
90
+ const loaded = talking ? await loadWorld(repoRoot, spec.talker.adapter) : undefined;
91
+ // A variant page is served at / with the site's own assets (fonts) around it.
92
+ const sandbox = talking ? undefined : options.targetOrigin
93
+ ? await startSandbox({ targetOrigin: options.targetOrigin, outsideRequests: options.outsideRequests })
94
+ : await startSandbox({
95
+ rootDir: path.dirname(path.resolve(repoRoot, spec.page ?? pagePath)),
96
+ indexFile: path.resolve(repoRoot, pagePath),
97
+ outsideRequests: options.outsideRequests,
98
+ });
99
+ async function worker() {
100
+ while (next < tasks.length) {
101
+ if (spent >= options.budgetUsd) {
102
+ budgetHit = true;
103
+ return;
104
+ }
105
+ const task = tasks[next++];
106
+ try {
107
+ let trace;
108
+ if (talking) {
109
+ // The engine grades the outcome from what the product did and, for a prose outcome, its own judge call.
110
+ trace = await runTalker({
111
+ spec, persona: task.persona, personaSet: task.set, seed: task.seed,
112
+ world: loaded.world, worldFile: loaded.file, target: options.targetOrigin, llm, batch,
113
+ });
114
+ spent += trace.cost.usd;
115
+ }
116
+ else if (spec.kind === 'task') {
117
+ // Visitors act on the app. The engine grades the outcome itself, so no judge call.
118
+ trace = await runVisitor({
119
+ repoRoot, spec, persona: task.persona, personaSet: task.set,
120
+ viewport: task.viewport, seed: task.seed, sandbox: sandbox, llm, batch, entryPath: options.entryPath,
121
+ });
122
+ spent += trace.cost.usd;
123
+ }
124
+ else {
125
+ const ungraded = await runReader({
126
+ repoRoot, spec, persona: task.persona, personaSet: task.set,
127
+ viewport: task.viewport, seed: task.seed, sandbox: sandbox, llm, batch, pagePath, entryPath: options.entryPath,
128
+ });
129
+ spent += ungraded.cost.usd; // count the persona call even if judging fails
130
+ trace = await gradeTrace(llm, spec.questions, ungraded);
131
+ spent += trace.cost.usd - ungraded.cost.usd;
132
+ }
133
+ validateTrace(trace, questionIds);
134
+ await writeTrace(tracesDir, trace);
135
+ traces.push(trace);
136
+ completed++;
137
+ }
138
+ catch (err) {
139
+ failed++;
140
+ log(`FAILED ${task.persona.id} ${task.viewport} seed ${task.seed}: ${String(err).slice(0, 900)}`);
141
+ }
142
+ const finished = completed + failed;
143
+ if (finished % 40 === 0 || finished === tasks.length)
144
+ log(` ${finished}/${tasks.length} finished, ${failed} failed, $${spent.toFixed(2)} spent`);
145
+ }
146
+ }
147
+ try {
148
+ await Promise.all(Array.from({ length: options.concurrency }, worker));
149
+ }
150
+ finally {
151
+ await sandbox?.finish(); // throws if any outside request was attempted
152
+ }
153
+ const blocked = options.outsideRequests === 'block'
154
+ ? blockedOrigins((sandbox?.violations ?? []).map((v) => v.url))
155
+ : [];
156
+ return { traces, newRuns: completed, failed, spentUsd: spent, budgetHit, blocked };
157
+ }
@@ -0,0 +1,56 @@
1
+ // Batch command line: every persona, working and holdout, N seeds per
2
+ // viewport, each run graded by the judge.
3
+ //
4
+ // node runner/batch.ts --batch <id> --budget-usd <n> [--page file | --target http://localhost:PORT [--entry /path]] [--runs 40]
5
+ // [--viewports mobile,desktop] [--personas all|id,id]
6
+ // [--sets working,holdout] [--concurrency 6] [--spec file]
7
+ import path from 'node:path';
8
+ import { parseArgs } from 'node:util';
9
+ import { runBatch } from "./batch-lib.js";
10
+ import { createLLM } from "./llm/index.js";
11
+ import { loadSpec } from "./spec.js";
12
+ const repoRoot = path.resolve(import.meta.dirname, '..');
13
+ const { values } = parseArgs({
14
+ options: {
15
+ spec: { type: 'string', default: 'specs/sdd-site.spec.yaml' },
16
+ batch: { type: 'string' },
17
+ page: { type: 'string' },
18
+ target: { type: 'string' },
19
+ entry: { type: 'string' },
20
+ runs: { type: 'string' },
21
+ viewports: { type: 'string' },
22
+ personas: { type: 'string', default: 'all' },
23
+ sets: { type: 'string', default: 'working,holdout' },
24
+ concurrency: { type: 'string', default: '6' },
25
+ 'budget-usd': { type: 'string' },
26
+ },
27
+ });
28
+ if (!values.batch || !values['budget-usd']) {
29
+ console.error('Usage: node runner/batch.ts --batch <id> --budget-usd <n> [--page file] [--runs n] [--viewports a,b] [--personas all|id,id] [--sets working,holdout] [--concurrency n]');
30
+ process.exit(2);
31
+ }
32
+ const spec = await loadSpec(path.resolve(repoRoot, values.spec));
33
+ const result = await runBatch({
34
+ repoRoot,
35
+ spec,
36
+ batch: values.batch,
37
+ pagePath: values.page,
38
+ targetOrigin: values.target,
39
+ entryPath: values.entry,
40
+ sets: values.sets.split(','),
41
+ personaIds: values.personas === 'all' ? undefined : values.personas.split(','),
42
+ viewports: (values.viewports ?? spec.entry.viewports.join(',')).split(','),
43
+ runs: Number(values.runs ?? spec.personas.runs_per_persona),
44
+ concurrency: Number(values.concurrency),
45
+ budgetUsd: Number(values['budget-usd']),
46
+ llm: createLLM(),
47
+ });
48
+ console.log(`Done: ${result.newRuns} new traces, ${result.failed} failed, $${result.spentUsd.toFixed(2)} spent on new runs`);
49
+ if (result.budgetHit) {
50
+ console.error('STOPPED: budget reached. Some runs were not started.');
51
+ process.exit(3);
52
+ }
53
+ if (result.failed > 0) {
54
+ console.error('Some runs failed. Rerun the same command to retry only those.');
55
+ process.exit(1);
56
+ }