sdd-engine 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +165 -2
  3. package/dist/engine/cli.js +95 -0
  4. package/dist/engine/simulate.js +214 -0
  5. package/dist/judge/conversation.js +45 -0
  6. package/dist/judge/judge.js +122 -0
  7. package/dist/judge/outcome.js +42 -0
  8. package/dist/judge/regrade.js +74 -0
  9. package/dist/pipeline/agent.js +78 -0
  10. package/dist/pipeline/agentstage.js +88 -0
  11. package/dist/pipeline/approve.js +68 -0
  12. package/dist/pipeline/cli.js +261 -0
  13. package/dist/pipeline/derive.js +287 -0
  14. package/dist/pipeline/designloop.js +385 -0
  15. package/dist/pipeline/facts.js +108 -0
  16. package/dist/pipeline/mock.js +253 -0
  17. package/dist/pipeline/project.js +79 -0
  18. package/dist/pipeline/render.js +95 -0
  19. package/dist/pipeline/spec.js +272 -0
  20. package/dist/pipeline/try.js +162 -0
  21. package/dist/runner/batch-lib.js +157 -0
  22. package/dist/runner/batch.js +56 -0
  23. package/dist/runner/cli.js +61 -0
  24. package/dist/runner/llm/claude-code.js +123 -0
  25. package/dist/runner/llm/index.js +12 -0
  26. package/dist/runner/llm/types.js +17 -0
  27. package/dist/runner/reader/answer.js +76 -0
  28. package/dist/runner/reader/extract.js +106 -0
  29. package/dist/runner/reader/run.js +111 -0
  30. package/dist/runner/reader/skim.js +115 -0
  31. package/dist/runner/rng.js +22 -0
  32. package/dist/runner/sandbox/cli.js +60 -0
  33. package/dist/runner/sandbox/sandbox.js +144 -0
  34. package/dist/runner/sandbox/server.js +64 -0
  35. package/dist/runner/spec.js +84 -0
  36. package/dist/runner/talker/decide.js +110 -0
  37. package/dist/runner/talker/run.js +159 -0
  38. package/dist/runner/talker/world.js +36 -0
  39. package/dist/runner/trace.js +130 -0
  40. package/dist/runner/visitor/decide.js +159 -0
  41. package/dist/runner/visitor/observe.js +102 -0
  42. package/dist/runner/visitor/run.js +296 -0
  43. package/dist/scorer/compare.js +119 -0
  44. package/dist/scorer/score.js +201 -0
  45. package/dist/scorer/spot-check-compare.js +41 -0
  46. package/dist/scorer/spot-check-view.js +45 -0
  47. package/dist/scorer/stats.js +45 -0
  48. package/dist/scorer/summary.js +62 -0
  49. package/package.json +49 -4
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Chad Cox (rangerchaz)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,166 @@
1
- # Temporary Holding Version
1
+ # sdd-engine
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ The engine for Simulation Driven Development (SDD).
4
+
5
+ A test asks "is the code correct?" and answers pass or fail. A simulation asks
6
+ "does this work for the people it is for?" and answers with a score against a
7
+ threshold. This engine sends simulated visitors through an app running on your
8
+ machine, scores what happened, and writes a report.
9
+
10
+ It only measures. It never changes the app.
11
+
12
+ **The scores are not calibrated.** The simulated visitors have not been checked
13
+ against real people. A score lets you compare two versions of the same app. It
14
+ is not a prediction of what real visitors will do.
15
+
16
+ ## Install
17
+
18
+ ```bash
19
+ npm install -g sdd-engine
20
+ sdd setup # downloads the browser the engine drives, once
21
+ ```
22
+
23
+ Needs Node 24 or later, and the [Claude Code](https://claude.com/claude-code)
24
+ CLI installed and logged in. The engine calls models through `claude -p`, so a
25
+ run is billed to whatever that CLI is logged in with.
26
+
27
+ ## Try it
28
+
29
+ ```bash
30
+ git clone https://github.com/TurkeyCode-ai/sdd && cd sdd
31
+ node examples/waitlist/app.ts & # a stand-in app on http://127.0.0.1:4180
32
+
33
+ sdd simulate --workdir examples/waitlist --target http://127.0.0.1:4180 \
34
+ --run demo --runs 2 --budget-usd 1
35
+ ```
36
+
37
+ The report lands in `examples/waitlist/reports/demo.md` and `demo.json`.
38
+
39
+ ## The command
40
+
41
+ ```
42
+ sdd simulate --workdir <dir> --target http://localhost:PORT --run <id> --budget-usd <n>
43
+ [--entry /path] [--runs n] [--concurrency n] [--outside-requests fail|block]
44
+ ```
45
+
46
+ | Flag | Meaning |
47
+ |---|---|
48
+ | `--workdir` | The work directory (below). Traces, screenshots and reports are written into it. |
49
+ | `--target` | The app to visit. It must be on this machine: `localhost`, `127.0.0.1` or `[::1]`. |
50
+ | `--run` | A name for this run. Using the same name again resumes it: finished visits are skipped. |
51
+ | `--budget-usd` | Stop starting new visits once this much is spent, at list prices. |
52
+ | `--entry` | The path visitors land on. Default `/`. |
53
+ | `--runs` | Visits per persona and viewport. Default: `personas.runs_per_persona` in the spec. |
54
+ | `--concurrency` | Visits at once. Default 6. |
55
+ | `--outside-requests` | `fail` (default): one request to another origin fails the run. `block`: such requests are refused and listed in the report, and the run carries on. Use `block` for apps that load fonts or scripts from elsewhere. Either way nothing outside the target is ever reached. |
56
+
57
+ Exit codes: `0` report written. `1` some visits failed, or the engine hit an
58
+ error; a report is written either way and says which. `2` bad arguments. `3`
59
+ budget reached; the report covers the visits that finished.
60
+
61
+ ## The work directory
62
+
63
+ ```
64
+ <workdir>/
65
+ spec.yaml what is simulated and what counts as passing
66
+ personas/working/*.yaml visitors whose results are reported in full
67
+ personas/holdout/*.yaml visitors held back; only their pass rates are reported
68
+ ```
69
+
70
+ The held-back set exists so that whoever revises the app cannot tune it to the
71
+ visitors they can see. Keep it away from them.
72
+
73
+ ### spec.yaml
74
+
75
+ Two kinds. **Reader mode** (`kind: comprehension`): visitors skim the entry page
76
+ for a fixed time, the page is taken away, and they answer questions from what
77
+ they actually took in. A separate judge grades each answer against a rubric.
78
+ See `examples/sdd-site/spec.yaml`.
79
+
80
+ **Visitor mode** (`kind: task`): visitors act on the app. They can scroll, click,
81
+ type, or leave. Leaving is the default: any other action has to quote words that
82
+ are really on the screen. The model is never told what success is. The engine
83
+ checks it from what the browser did. See `examples/waitlist/spec.yaml`.
84
+
85
+ ```yaml
86
+ experiment: waitlist-example
87
+ kind: task
88
+ goal: A visitor lands from an ad and joins the waitlist.
89
+ entry:
90
+ from: a social media ad
91
+ viewports: [mobile] # mobile, desktop
92
+ task:
93
+ goal: Join the waitlist
94
+ instruction: ... # optional: what the visitor was asked to try
95
+ success: # any of these; never shown to the model
96
+ request: { method: POST, path_includes: /api/waitlist }
97
+ url_includes: /thanks
98
+ text_visible: You are on the list
99
+ leaves_to: [github.com/acme/widget] # follows an outside link to here; the request is never made
100
+ clicks: [Copy] # presses a control whose label includes this
101
+ outcome: A confirmation that the visitor is on the list # judged from the final screen
102
+ visitor:
103
+ patience_seconds: 75
104
+ max_steps: 15
105
+ screenshots: true
106
+ personas:
107
+ runs_per_persona: 10
108
+ pass:
109
+ task_rate: 0.25 # one <question id>_rate per question; a task's id is "task"
110
+ ```
111
+
112
+ Visitors in task mode really submit forms. Point the engine only at a copy of
113
+ the app that is safe to act on.
114
+
115
+ ### Personas
116
+
117
+ ```yaml
118
+ id: collector
119
+ archetype: A man in his forties who collects sci-fi props
120
+ profile:
121
+ role: IT manager
122
+ experience: Builds replica props, comfortable with electronics kits
123
+ tech_comfort: high
124
+ already_uses: [3D printer, microcontroller kits]
125
+ arrival:
126
+ source: Instagram ad
127
+ saw: "The robot sidekick you wanted as a kid" # all they saw before landing
128
+ needs: [what it looks like, whether it can be modified]
129
+ objections: [AI gadgets oversell, backed projects that never shipped]
130
+ dials: { reading_wpm: 270, skim_rate: 0.5 } # mis_tap_rate is optional, default 0.03
131
+ ```
132
+
133
+ ## The report
134
+
135
+ `reports/<run>.json` has `schema: "sdd-report/1"`.
136
+
137
+ - `verdict`: `pass`, `not met`, `inside the noise`, or `incomplete`.
138
+ - A threshold is met only when the low end of its 95% interval clears it, on
139
+ both the working set and the held-back set. A few visits rarely settle
140
+ anything, and the report says so plainly.
141
+ - `working` has the detail: per question, per persona, what was missed, where
142
+ visitors stopped, and a sample of failing answers. `holdout` has rates only.
143
+ - `costUsd` is the spend at list prices. `blockedRequests` lists outside origins
144
+ that were refused. `error` is present when the engine could not finish.
145
+
146
+ ## Models
147
+
148
+ Chosen by environment variable, with defaults:
149
+
150
+ | Variable | Role | Default |
151
+ |---|---|---|
152
+ | `SDD_MODEL_PERSONA` | the visitors | `claude-haiku-5-5` |
153
+ | `SDD_MODEL_JUDGE` | grades answers and outcomes | `claude-sonnet-5-5` |
154
+
155
+ ## Develop
156
+
157
+ ```bash
158
+ npm install
159
+ npm run typecheck
160
+ npm test # offline: the model is scripted
161
+ npm run build # compiles to dist/, which is what the published command runs
162
+ ```
163
+
164
+ ## License
165
+
166
+ MIT
@@ -0,0 +1,95 @@
1
+ #!/usr/bin/env node
2
+ // The `sdd` command.
3
+ //
4
+ // sdd simulate ... run simulated visitors against a local app and write a report
5
+ // sdd setup download the browser the engine drives
6
+ // sdd --version
7
+ // sdd --help
8
+ import { spawnSync } from 'node:child_process';
9
+ import { readFile } from 'node:fs/promises';
10
+ import { createRequire } from 'node:module';
11
+ import path from 'node:path';
12
+ import { approveCommand, deriveCommand, designCommand, reviseCommand, scoreCommand, showCommand } from "../pipeline/cli.js";
13
+ import { simulate, USAGE } from "./simulate.js";
14
+ const HELP = `sdd: Simulation Driven Development engine
15
+
16
+ sdd simulate --workdir <dir> --target http://localhost:PORT --run <id> --budget-usd <n>
17
+ [--entry /path] [--runs n] [--concurrency n] [--outside-requests fail|block]
18
+ sdd setup download the browser the engine drives (once, after installing)
19
+ sdd --version
20
+
21
+ sdd derive "<the result you want>" [--source <folder or address>]... [--budget-usd 5]
22
+ turn one sentence into a spec: who it is for, what counts as
23
+ success, thresholds, held-back people, assumptions, and a
24
+ ledger of true claims. Nothing is simulated or built.
25
+ sdd revise <project> "<what to change>" the owner's corrections, applied by the deriver
26
+ sdd approve <project> --budget-usd <n> the owner approves this exact spec
27
+ sdd design <project> [--iterations 6] [--runs 4] [--try 5] design on mocks, score, revise
28
+ sdd score <project> --page <folder> --name <label> score a page the pipeline did not design
29
+ (the site as it stands, say) with the same people and checks
30
+ sdd show [<project>] list projects, or print one project's derived spec
31
+
32
+ The work directory holds spec.yaml, personas/working/*.yaml and
33
+ personas/holdout/*.yaml. The report is written to <workdir>/reports/<run>.json
34
+ and .md. The target must be an app running on this machine; nothing outside it
35
+ is ever reached.
36
+
37
+ Exit codes: 0 report written, 1 runs failed or the engine hit an error (the
38
+ report says which), 2 bad arguments, 3 budget reached.
39
+
40
+ Needs: Node 24 or later, the browser from "sdd setup", and the claude CLI
41
+ logged in.`;
42
+ async function version() {
43
+ // package.json sits one level above this file in the source tree and two
44
+ // levels above it once built into dist/engine.
45
+ for (const up of ['../package.json', '../../package.json']) {
46
+ try {
47
+ const pkg = JSON.parse(await readFile(new URL(up, import.meta.url), 'utf8'));
48
+ if (pkg.version)
49
+ return pkg.version;
50
+ }
51
+ catch { /* try the next location */ }
52
+ }
53
+ return 'unknown';
54
+ }
55
+ const [command, ...rest] = process.argv.slice(2);
56
+ if (command === '--version' || command === '-v') {
57
+ console.log(await version());
58
+ }
59
+ else if (command === 'setup') {
60
+ // Download the browser build this engine's own Playwright expects. Running
61
+ // "npx playwright install" from somewhere else can fetch a different build.
62
+ const playwrightCli = path.join(path.dirname(createRequire(import.meta.url).resolve('playwright/package.json')), 'cli.js');
63
+ const result = spawnSync(process.execPath, [playwrightCli, 'install', 'chromium'], { stdio: 'inherit' });
64
+ process.exit(result.status ?? 1);
65
+ }
66
+ else if (command === 'simulate') {
67
+ // exit() and not exitCode: a browser or child process that is slow to close must not hold the command open.
68
+ process.exit(await simulate(rest));
69
+ }
70
+ else if (command === 'derive') {
71
+ process.exit(await deriveCommand(rest));
72
+ }
73
+ else if (command === 'revise') {
74
+ process.exit(await reviseCommand(rest));
75
+ }
76
+ else if (command === 'approve') {
77
+ process.exit(await approveCommand(rest));
78
+ }
79
+ else if (command === 'design') {
80
+ process.exit(await designCommand(rest));
81
+ }
82
+ else if (command === 'score') {
83
+ process.exit(await scoreCommand(rest));
84
+ }
85
+ else if (command === 'show') {
86
+ process.exit(await showCommand(rest));
87
+ }
88
+ else if (command === undefined || command === '--help' || command === '-h' || command === 'help') {
89
+ console.log(HELP);
90
+ }
91
+ else {
92
+ console.error(`Unknown command: ${command}`);
93
+ console.error(USAGE);
94
+ process.exitCode = 2;
95
+ }
@@ -0,0 +1,214 @@
1
+ // The engine's single entry point for a host product such as TurkeyCode.
2
+ //
3
+ // sdd simulate --workdir <dir> --target http://localhost:PORT
4
+ // --run <id> --budget-usd <n>
5
+ // [--entry /path] [--runs n] [--concurrency 6]
6
+ // [--outside-requests fail|block]
7
+ //
8
+ // The work directory holds everything for one app:
9
+ //
10
+ // <workdir>/spec.yaml the simulation spec
11
+ // <workdir>/personas/working/*.yaml personas the builder may see
12
+ // <workdir>/personas/holdout/*.yaml personas held back
13
+ //
14
+ // The engine writes, under the same directory:
15
+ //
16
+ // traces/<experiment>/*.json one per persona run, never edited
17
+ // shots/... screenshots the personas were shown
18
+ // reports/<run>.json the result a product renders
19
+ // reports/<run>.md the same, for a person to read
20
+ //
21
+ // Report only. It measures and reports. It changes nothing in the app.
22
+ // The target must be an app running on this machine. Every request to any
23
+ // other origin is blocked. By default one attempt fails the run. With
24
+ // --outside-requests block the run carries on and the report lists what was
25
+ // refused, for apps that load fonts or scripts from elsewhere.
26
+ //
27
+ // Exit codes: 0 report written, 1 some runs failed or the engine hit an error
28
+ // (a report is written either way; an error gives verdict "incomplete"),
29
+ // 2 bad arguments, 3 budget reached (report covers the runs that finished).
30
+ import { mkdir, readFile, writeFile } from 'node:fs/promises';
31
+ import path from 'node:path';
32
+ import { parseArgs } from 'node:util';
33
+ import { parse } from 'yaml';
34
+ import { runBatch } from "../runner/batch-lib.js";
35
+ import { createLLM } from "../runner/llm/index.js";
36
+ import { loadSpec } from "../runner/spec.js";
37
+ import { summarize } from "../scorer/summary.js";
38
+ export const USAGE = 'Usage: sdd simulate --workdir <dir> --target http://localhost:PORT --run <id> --budget-usd <n> ' +
39
+ '[--entry /path] [--runs n] [--concurrency n] [--outside-requests fail|block]';
40
+ /** Runs one simulation and returns the process exit code. */
41
+ export async function simulate(args, deps = {}) {
42
+ const log = deps.log ?? console.log;
43
+ const logError = deps.logError ?? console.error;
44
+ let values;
45
+ try {
46
+ ({ values } = parseArgs({
47
+ args,
48
+ options: {
49
+ workdir: { type: 'string' },
50
+ target: { type: 'string' },
51
+ entry: { type: 'string', default: '/' },
52
+ run: { type: 'string' },
53
+ runs: { type: 'string' },
54
+ concurrency: { type: 'string', default: '6' },
55
+ 'budget-usd': { type: 'string' },
56
+ 'outside-requests': { type: 'string', default: 'fail' },
57
+ },
58
+ }));
59
+ }
60
+ catch (err) {
61
+ logError(err instanceof Error ? err.message : String(err));
62
+ logError(USAGE);
63
+ return 2;
64
+ }
65
+ if (!values.workdir || !values.target || !values.run || !values['budget-usd']) {
66
+ logError(USAGE);
67
+ return 2;
68
+ }
69
+ const outsideRequests = values['outside-requests'];
70
+ if (outsideRequests !== 'fail' && outsideRequests !== 'block') {
71
+ logError('--outside-requests must be "fail" or "block"');
72
+ return 2;
73
+ }
74
+ const run = values.run;
75
+ const target = `${values.target}${values.entry}`;
76
+ const workDir = path.resolve(values.workdir);
77
+ const outDir = path.join(workDir, 'reports');
78
+ const reportFile = path.join(outDir, `${run}.json`);
79
+ const startedAt = new Date().toISOString();
80
+ try {
81
+ const specFile = path.join(workDir, 'spec.yaml');
82
+ const spec = await loadSpec(specFile);
83
+ // Inside a work directory the persona folders are fixed. The spec cannot point elsewhere.
84
+ spec.personas = { ...spec.personas, working: 'personas/working', holdout: 'personas/holdout' };
85
+ const rawPass = parse(await readFile(specFile, 'utf8')).pass ?? {};
86
+ const thresholds = {};
87
+ for (const q of spec.questions) {
88
+ const t = Number(rawPass[`${q.id}_rate`]);
89
+ if (!(t > 0 && t <= 1))
90
+ throw new Error(`spec.yaml: pass.${q.id}_rate must be a number between 0 and 1`);
91
+ thresholds[q.id] = t;
92
+ }
93
+ const result = await runBatch({
94
+ repoRoot: workDir,
95
+ spec,
96
+ batch: run,
97
+ targetOrigin: values.target,
98
+ entryPath: values.entry,
99
+ sets: ['working', 'holdout'],
100
+ viewports: spec.entry.viewports,
101
+ runs: Number(values.runs ?? spec.personas.runs_per_persona),
102
+ concurrency: Number(values.concurrency),
103
+ budgetUsd: Number(values['budget-usd']),
104
+ outsideRequests,
105
+ llm: deps.llm ?? createLLM(),
106
+ log,
107
+ });
108
+ const working = summarize(spec, thresholds, result.traces, 'working');
109
+ const holdout = summarize(spec, thresholds, result.traces, 'holdout');
110
+ const complete = result.failed === 0 && !result.budgetHit;
111
+ const costUsd = result.traces.reduce((sum, t) => sum + t.cost.usd, 0);
112
+ const calibrated = result.traces.length > 0 && result.traces.every((t) => t.dials.calibrated);
113
+ // "pass" needs every lower bound to clear on both sets. Anything else is
114
+ // reported as it is: not met, or not enough runs to say.
115
+ const verdict = !complete ? 'incomplete'
116
+ : working.met && holdout.met ? 'pass'
117
+ : [...working.questions, ...holdout.questions].some((q) => q.verdict === 'not met') ? 'not met'
118
+ : 'inside the noise';
119
+ /** Holdout detail stays out of anything a builder might read: rates only. */
120
+ const strip = (s) => s.set === 'working'
121
+ ? s
122
+ : { set: s.set, runs: s.runs, questions: s.questions, shortfall: s.shortfall, minMargin: s.minMargin, met: s.met, ratesClear: s.ratesClear };
123
+ const report = {
124
+ schema: 'sdd-report/1',
125
+ run,
126
+ experiment: spec.experiment,
127
+ goal: spec.goal,
128
+ target,
129
+ startedAt,
130
+ finishedAt: new Date().toISOString(),
131
+ verdict,
132
+ calibrated,
133
+ runs: { completed: result.traces.length, failed: result.failed, budgetHit: result.budgetHit },
134
+ costUsd: Math.round(costUsd * 10000) / 10000,
135
+ blockedRequests: result.blocked,
136
+ page: result.traces[0]?.page,
137
+ questions: spec.questions.map((q) => ({ id: q.id, ask: q.ask })),
138
+ working: strip(working),
139
+ holdout: strip(holdout),
140
+ };
141
+ const pct = (x) => (Number.isNaN(x) ? 'n/a' : `${Math.round(x * 100)}%`);
142
+ const table = (s) => ['| Question | Passed | 95% interval | Threshold | Verdict |', '|---|---|---|---|---|',
143
+ ...s.questions.map((q) => `| ${q.id} | ${pct(q.interval.rate)} (${q.interval.passed}/${q.interval.n}) | ${pct(q.interval.lower)} to ${pct(q.interval.upper)} | ${pct(q.threshold)} | ${q.verdict} |`)].join('\n');
144
+ const blockedLine = result.blocked.length === 0
145
+ ? ''
146
+ : `- Outside requests refused, so visitors saw the app without them: ${result.blocked.map((b) => `${b.origin} (${b.count})`).join(', ')}\n`;
147
+ const md = `# Simulation report: ${spec.experiment}, run "${run}"
148
+
149
+ **Verdict: ${verdict}.** A threshold is met only when the low end of its 95% interval clears it, for both the working readers and the held-back readers.
150
+ ${calibrated ? '' : '\n**Not calibrated.** These are simulated visitors that have not been checked against real ones. The scores compare versions of this page with each other. They are not a prediction about real visitors.\n'}
151
+ - Goal: ${spec.goal.trim()}
152
+ - Target: ${target}
153
+ - Runs: ${result.traces.length} completed, ${result.failed} failed${result.budgetHit ? ', stopped by the budget' : ''}
154
+ - Cost at list prices: $${costUsd.toFixed(2)}
155
+ ${blockedLine}
156
+ ## Working readers
157
+
158
+ ${table(working)}
159
+
160
+ ## Held-back readers
161
+
162
+ ${table(holdout)}
163
+
164
+ ## What readers missed most
165
+
166
+ ${working.missedElements.map((m) => `- ${m.question}: "${m.element}" missed by ${pct(m.missedRate)}`).join('\n')}
167
+
168
+ ## Where readers were when time ran out
169
+
170
+ ${Object.entries(working.stoppedIn).map(([name, x]) => `- ${name}: ${pct(x)}`).join('\n')}
171
+
172
+ ## A sample of failing answers
173
+
174
+ ${working.failingAnswers.map((f) => `- [${f.question}] ${f.answer}\n - Judge: ${f.judgeNote}`).join('\n')}
175
+ `;
176
+ await mkdir(outDir, { recursive: true });
177
+ await writeFile(reportFile, JSON.stringify(report, null, 2) + '\n');
178
+ await writeFile(path.join(outDir, `${run}.md`), md);
179
+ log(`Verdict: ${verdict}. Report: ${reportFile}`);
180
+ return result.budgetHit ? 3 : result.failed > 0 ? 1 : 0;
181
+ }
182
+ catch (err) {
183
+ // Exit 1 always leaves a report, so a host never has to guess what happened.
184
+ const raw = err instanceof Error ? err.message : String(err);
185
+ // Playwright's own advice ("npx playwright install") can fetch a build
186
+ // that does not match this engine. Point at the command that does.
187
+ const error = /Executable doesn't exist/.test(raw)
188
+ ? `The browser the engine drives is not installed. Run: sdd setup\n(${raw.split('\n')[0]})`
189
+ : raw;
190
+ const empty = (set) => ({ set, runs: 0, questions: [], met: false });
191
+ const report = {
192
+ schema: 'sdd-report/1',
193
+ run,
194
+ target,
195
+ startedAt,
196
+ finishedAt: new Date().toISOString(),
197
+ verdict: 'incomplete',
198
+ calibrated: false,
199
+ error,
200
+ runs: { completed: 0, failed: 0, budgetHit: false },
201
+ costUsd: 0,
202
+ blockedRequests: [],
203
+ questions: [],
204
+ working: empty('working'),
205
+ holdout: empty('holdout'),
206
+ };
207
+ await mkdir(outDir, { recursive: true });
208
+ await writeFile(reportFile, JSON.stringify(report, null, 2) + '\n');
209
+ await writeFile(path.join(outDir, `${run}.md`), `# Simulation report: run "${run}"\n\n**Verdict: incomplete.** The engine stopped before any result could be scored.\n\n ${error.split('\n').join('\n ')}\n`);
210
+ logError(`Simulation failed: ${error}`);
211
+ logError(`Report: ${reportFile}`);
212
+ return 1;
213
+ }
214
+ }
@@ -0,0 +1,45 @@
1
+ // Judges whether a spoken conversation ended with the expected outcome.
2
+ // Part of the referee.
3
+ //
4
+ // The judge sees the expected outcome and the transcript: what the person
5
+ // said, what the product said back, and what the product did (its events).
6
+ // It does not see the persona, the persona model's reasoning, or whether the
7
+ // person believed it worked.
8
+ const SYSTEM = `You check whether a conversation between a person and a voice assistant ended with the expected outcome. You are strict.
9
+
10
+ You get the expected outcome and the transcript. Each turn has what the person said, what the assistant said back, and a list of what the assistant actually did (tools it ran). Decide whether the transcript shows the expected outcome.
11
+
12
+ Rules:
13
+ - Evidence first. Quote the exact words or name the exact action in the transcript that show the outcome. If there are none, write "none". Decide only after quoting.
14
+ - Saying is not doing. If the outcome needs an action and the assistant only said it did it, with no matching entry under what it did, the outcome is not met.
15
+ - A promise to do it later, a question back to the person, or an offer is not the outcome.
16
+ - A wrong name, wrong amount, or wrong thing is not the outcome, even if the assistant sounds confident.
17
+ - Do not give credit for what probably happened. Judge only what the transcript shows.
18
+ - The note is one short sentence: what shows the outcome, or what is missing.`;
19
+ export function buildConversationRequest(expected, when, turns) {
20
+ const transcript = turns
21
+ .map((t, i) => `Turn ${i + 1}\n Person: "${t.said}"\n Assistant: ${t.heard ? `"${t.heard}"` : '(said nothing)'}\n Assistant did: ${t.events.length ? t.events.map((e) => (e.detail ? `${e.name} ${e.detail}` : e.name)).join('; ') : '(nothing)'}`)
22
+ .join('\n');
23
+ return {
24
+ role: 'judge',
25
+ system: SYSTEM,
26
+ prompt: `EXPECTED OUTCOME\n${expected}\n\nWHEN\nThe conversation took place on ${when}, local time. Times in the assistant's actions may be written in UTC.\n\nTRANSCRIPT\n${transcript || '(no turns)'}\n\nDoes the transcript show the expected outcome?`,
27
+ schema: {
28
+ type: 'object',
29
+ properties: { evidence: { type: 'string' }, met: { type: 'boolean' }, note: { type: 'string' } },
30
+ required: ['evidence', 'met', 'note'],
31
+ },
32
+ };
33
+ }
34
+ export async function judgeConversation(llm, expected, when, turns) {
35
+ const request = buildConversationRequest(expected, when, turns);
36
+ const calls = [];
37
+ for (let attempt = 0; attempt < 2; attempt++) {
38
+ const response = await llm.complete(request);
39
+ calls.push({ request, response });
40
+ const reply = response.structured;
41
+ if (reply && typeof reply.met === 'boolean')
42
+ return { met: reply.met, evidence: reply.evidence ?? '', note: reply.note ?? '', calls };
43
+ }
44
+ throw new Error('The conversation judge gave no verdict');
45
+ }
@@ -0,0 +1,122 @@
1
+ // The judge. Part of the referee: the revising agent may never edit this.
2
+ //
3
+ // It sees the questions, the answers, and the rubric. Nothing else. It does
4
+ // not get the page, the persona, the viewport, the seed, or anything that
5
+ // says which page variant produced the answers.
6
+ //
7
+ // Version 2. Version 1 gave a verdict first and a note after, and a blind
8
+ // second review agreed with it on only 64% of contrast answers. Its notes
9
+ // sometimes contradicted its own verdicts. Version 2 must quote the evidence
10
+ // for each element before it decides.
11
+ export const JUDGE_VERSION = 'v2';
12
+ const SYSTEM = `You grade short written answers against a rubric. You are careful and consistent.
13
+
14
+ For each question you get the question, one person's answer, and a list of rubric elements. For each element, first quote the words from the answer that bear on it, then decide whether the answer meets it.
15
+
16
+ Rules:
17
+ - Evidence first. Copy the relevant words from the answer exactly. If there are none, write "none". Decide only after quoting.
18
+ - Grade the meaning, not the wording. A clear paraphrase meets an element.
19
+ - An element with several parts is met only if every part is stated. For example "a test checks whether code is correct, pass or fail" needs both that a test checks the code or its correctness, and that the result is binary (pass or fail, passing or failing, or the like). An example assertion on its own does not state a binary result.
20
+ - A part can be stated by contrast. "A score against a threshold rather than a pass or fail" states that the other side is pass or fail.
21
+ - Do not give credit for what the person probably meant, and do not use your own knowledge of the subject to fill gaps.
22
+ - People hedge. A claim counts if the person states it, even if they add that they are unsure about other details or about how the page framed it. A claim does not count if the person says they do not know it or cannot remember it.
23
+ - An element that asks for "at least N of" a list is met if at least N of the listed items are named, in any wording or letter case. Being unsure of their order does not matter.
24
+ - Grade each question on its own answer only. Do not carry information from one answer to another.
25
+ - The note is one short sentence saying what was missing, or "complete". It must agree with your decisions.`;
26
+ export function buildJudgeRequest(questions, answers) {
27
+ const body = questions
28
+ .map((q) => {
29
+ const answer = answers.find((a) => a.id === q.id);
30
+ if (!answer)
31
+ throw new Error(`No answer to grade for question "${q.id}"`);
32
+ const required = q.rubric.required.map((r, i) => ` required ${i + 1}: ${r}`).join('\n');
33
+ const bonus = (q.rubric.bonus ?? []).map((r, i) => ` bonus ${i + 1}: ${r}`).join('\n');
34
+ return `QUESTION ${q.id}: ${q.ask}\nANSWER: ${answer.answer}\nRUBRIC:\n${required}${bonus ? `\n${bonus}` : ''}`;
35
+ })
36
+ .join('\n\n');
37
+ const element = {
38
+ type: 'object',
39
+ properties: { evidence: { type: 'string' }, met: { type: 'boolean' } },
40
+ required: ['evidence', 'met'],
41
+ };
42
+ return {
43
+ role: 'judge',
44
+ system: SYSTEM,
45
+ prompt: `${body}\n\nGrade every question by its id: ${questions.map((q) => q.id).join(', ')}. Return exactly ${questions.length} grades. For each, give one entry per rubric element, in the order listed.`,
46
+ schema: {
47
+ type: 'object',
48
+ properties: {
49
+ grades: {
50
+ type: 'array',
51
+ minItems: questions.length,
52
+ maxItems: questions.length,
53
+ items: {
54
+ type: 'object',
55
+ properties: {
56
+ id: { type: 'string' },
57
+ required: { type: 'array', items: element },
58
+ bonus: { type: 'array', items: element },
59
+ note: { type: 'string' },
60
+ },
61
+ required: ['id', 'required', 'bonus', 'note'],
62
+ },
63
+ },
64
+ },
65
+ required: ['grades'],
66
+ },
67
+ };
68
+ }
69
+ export async function judgeAnswers(llm, questions, answers) {
70
+ const request = buildJudgeRequest(questions, answers);
71
+ const calls = [];
72
+ let problem = '';
73
+ // The judge sometimes returns grades for only some of the questions. Ask
74
+ // again with the same request, never a changed one. Every attempt is kept
75
+ // and paid for.
76
+ for (let attempt = 0; attempt < 3; attempt++) {
77
+ const response = await llm.complete(request);
78
+ calls.push({ request, response });
79
+ const reply = response.structured;
80
+ const grades = [];
81
+ problem = '';
82
+ for (const q of questions) {
83
+ const g = reply?.grades?.find((x) => x.id === q.id);
84
+ const bonusCount = q.rubric.bonus?.length ?? 0;
85
+ const wellFormed = g &&
86
+ g.required?.length === q.rubric.required.length &&
87
+ (g.bonus?.length ?? 0) >= bonusCount &&
88
+ [...g.required, ...g.bonus].every((e) => typeof e?.met === 'boolean');
89
+ if (!g || !wellFormed) {
90
+ problem = `Judge reply is missing or malformed for question "${q.id}". Full reply: ${JSON.stringify(reply).slice(0, 600)}`;
91
+ break;
92
+ }
93
+ grades.push({
94
+ questionId: q.id,
95
+ pass: g.required.every((e) => e.met), // a pass needs every required element
96
+ requiredMet: g.required.map((e) => e.met),
97
+ bonusMet: g.bonus.slice(0, bonusCount).map((e) => e.met),
98
+ evidence: g.required.map((e) => e.evidence),
99
+ note: g.note,
100
+ });
101
+ }
102
+ if (!problem)
103
+ return { grades, calls };
104
+ }
105
+ throw new Error(problem);
106
+ }
107
+ /** Returns a graded copy of a not yet stored trace, with the judge calls and their cost added. */
108
+ export async function gradeTrace(llm, questions, trace) {
109
+ const { grades, calls } = await judgeAnswers(llm, questions, trace.answers);
110
+ const llmCalls = [...trace.llmCalls, ...calls];
111
+ const tokens = (c) => c.response.usage.inputTokens + c.response.usage.cacheReadTokens + c.response.usage.cacheWriteTokens;
112
+ return {
113
+ ...trace,
114
+ grades,
115
+ llmCalls,
116
+ cost: {
117
+ usd: llmCalls.reduce((sum, c) => sum + c.response.costUsd, 0),
118
+ inputTokens: llmCalls.reduce((sum, c) => sum + tokens(c), 0),
119
+ outputTokens: llmCalls.reduce((sum, c) => sum + c.response.usage.outputTokens, 0),
120
+ },
121
+ };
122
+ }