sdd-engine 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +178 -2
  3. package/dist/engine/cli.js +95 -0
  4. package/dist/engine/simulate.js +214 -0
  5. package/dist/judge/conversation.js +45 -0
  6. package/dist/judge/judge.js +122 -0
  7. package/dist/judge/outcome.js +42 -0
  8. package/dist/judge/regrade.js +74 -0
  9. package/dist/pipeline/agent.js +78 -0
  10. package/dist/pipeline/agentstage.js +88 -0
  11. package/dist/pipeline/approve.js +68 -0
  12. package/dist/pipeline/cli.js +261 -0
  13. package/dist/pipeline/derive.js +287 -0
  14. package/dist/pipeline/designloop.js +385 -0
  15. package/dist/pipeline/facts.js +108 -0
  16. package/dist/pipeline/mock.js +253 -0
  17. package/dist/pipeline/project.js +79 -0
  18. package/dist/pipeline/render.js +95 -0
  19. package/dist/pipeline/spec.js +272 -0
  20. package/dist/pipeline/try.js +162 -0
  21. package/dist/runner/batch-lib.js +157 -0
  22. package/dist/runner/batch.js +56 -0
  23. package/dist/runner/cli.js +61 -0
  24. package/dist/runner/llm/claude-code.js +123 -0
  25. package/dist/runner/llm/index.js +12 -0
  26. package/dist/runner/llm/types.js +17 -0
  27. package/dist/runner/reader/answer.js +76 -0
  28. package/dist/runner/reader/extract.js +106 -0
  29. package/dist/runner/reader/run.js +111 -0
  30. package/dist/runner/reader/skim.js +115 -0
  31. package/dist/runner/rng.js +22 -0
  32. package/dist/runner/sandbox/cli.js +60 -0
  33. package/dist/runner/sandbox/sandbox.js +144 -0
  34. package/dist/runner/sandbox/server.js +64 -0
  35. package/dist/runner/spec.js +84 -0
  36. package/dist/runner/talker/decide.js +110 -0
  37. package/dist/runner/talker/run.js +159 -0
  38. package/dist/runner/talker/world.js +36 -0
  39. package/dist/runner/trace.js +130 -0
  40. package/dist/runner/visitor/decide.js +159 -0
  41. package/dist/runner/visitor/observe.js +102 -0
  42. package/dist/runner/visitor/run.js +296 -0
  43. package/dist/scorer/compare.js +119 -0
  44. package/dist/scorer/score.js +201 -0
  45. package/dist/scorer/spot-check-compare.js +41 -0
  46. package/dist/scorer/spot-check-view.js +45 -0
  47. package/dist/scorer/stats.js +45 -0
  48. package/dist/scorer/summary.js +62 -0
  49. package/package.json +49 -4
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Chad Cox (rangerchaz)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,179 @@
1
- # Temporary Holding Version
1
+ # sdd-engine
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ The engine for Simulation Driven Development (SDD).
4
+
5
+ A test asks "is the code correct?" and answers pass or fail. A simulation asks
6
+ "does this work for the people it is for?" and answers with a score against a
7
+ threshold. This engine sends simulated visitors through an app running on your
8
+ machine, scores what happened, and writes a report.
9
+
10
+ It only measures. It never changes the app.
11
+
12
+ **The scores are not calibrated.** The simulated visitors have not been checked
13
+ against real people. A score lets you compare two versions of the same app. It
14
+ is not a prediction of what real visitors will do.
15
+
16
+ ## Install
17
+
18
+ ```bash
19
+ npm install -g sdd-engine
20
+ sdd setup # downloads the browser the engine drives, once
21
+ ```
22
+
23
+ If the global install fails with `EACCES`, your user cannot write to npm's
24
+ global folder. That is the default on Linux with the system Node and on the
25
+ official Docker image. Run it with `npx` instead, which needs no install:
26
+
27
+ ```bash
28
+ npx sdd-engine setup
29
+ npx sdd-engine --help
30
+ ```
31
+
32
+ Needs Node 20.11 or later, and the [Claude Code](https://claude.com/claude-code)
33
+ CLI installed and logged in. The engine calls models through `claude -p`, so a
34
+ run is billed to whatever that CLI is logged in with.
35
+
36
+ ## Try it
37
+
38
+ ```bash
39
+ git clone https://github.com/TurkeyCode-ai/sdd && cd sdd
40
+ node examples/waitlist/app.ts & # a stand-in app on http://127.0.0.1:4180
41
+
42
+ sdd simulate --workdir examples/waitlist --target http://127.0.0.1:4180 \
43
+ --run demo --runs 2 --budget-usd 1
44
+ ```
45
+
46
+ The report lands in `examples/waitlist/reports/demo.md` and `demo.json`.
47
+
48
+ ## The command
49
+
50
+ ```
51
+ sdd simulate --workdir <dir> --target http://localhost:PORT --run <id> --budget-usd <n>
52
+ [--entry /path] [--runs n] [--concurrency n] [--outside-requests fail|block]
53
+ ```
54
+
55
+ | Flag | Meaning |
56
+ |---|---|
57
+ | `--workdir` | The work directory (below). Traces, screenshots and reports are written into it. |
58
+ | `--target` | The app to visit. It must be on this machine: `localhost`, `127.0.0.1` or `[::1]`. |
59
+ | `--run` | A name for this run. Using the same name again resumes it: finished visits are skipped. |
60
+ | `--budget-usd` | Stop starting new visits once this much is spent, at list prices. |
61
+ | `--entry` | The path visitors land on. Default `/`. |
62
+ | `--runs` | Visits per persona and viewport. Default: `personas.runs_per_persona` in the spec. |
63
+ | `--concurrency` | Visits at once. Default 6. |
64
+ | `--outside-requests` | `fail` (default): one request to another origin fails the run. `block`: such requests are refused and listed in the report, and the run carries on. Use `block` for apps that load fonts or scripts from elsewhere. Either way nothing outside the target is ever reached. |
65
+
66
+ Exit codes: `0` report written. `1` some visits failed, or the engine hit an
67
+ error; a report is written either way and says which. `2` bad arguments. `3`
68
+ budget reached; the report covers the visits that finished.
69
+
70
+ ## The work directory
71
+
72
+ ```
73
+ <workdir>/
74
+ spec.yaml what is simulated and what counts as passing
75
+ personas/working/*.yaml visitors whose results are reported in full
76
+ personas/holdout/*.yaml visitors held back; only their pass rates are reported
77
+ ```
78
+
79
+ The held-back set exists so that whoever revises the app cannot tune it to the
80
+ visitors they can see. Keep it away from them.
81
+
82
+ ### spec.yaml
83
+
84
+ Two kinds. **Reader mode** (`kind: comprehension`): visitors skim the entry page
85
+ for a fixed time, the page is taken away, and they answer questions from what
86
+ they actually took in. A separate judge grades each answer against a rubric.
87
+ See `examples/sdd-site/spec.yaml`.
88
+
89
+ **Visitor mode** (`kind: task`): visitors act on the app. They can scroll, click,
90
+ type, or leave. Leaving is the default: any other action has to quote words that
91
+ are really on the screen. The model is never told what success is. The engine
92
+ checks it from what the browser did. See `examples/waitlist/spec.yaml`.
93
+
94
+ ```yaml
95
+ experiment: waitlist-example
96
+ kind: task
97
+ goal: A visitor lands from an ad and joins the waitlist.
98
+ entry:
99
+ from: a social media ad
100
+ viewports: [mobile] # mobile, desktop
101
+ task:
102
+ goal: Join the waitlist
103
+ instruction: ... # optional: what the visitor was asked to try
104
+ success: # any of these; never shown to the model
105
+ request: { method: POST, path_includes: /api/waitlist }
106
+ url_includes: /thanks
107
+ text_visible: You are on the list
108
+ leaves_to: [github.com/acme/widget] # follows an outside link to here; the request is never made
109
+ clicks: [Copy] # presses a control whose label includes this
110
+ outcome: A confirmation that the visitor is on the list # judged from the final screen
111
+ visitor:
112
+ patience_seconds: 75
113
+ max_steps: 15
114
+ screenshots: true
115
+ personas:
116
+ runs_per_persona: 10
117
+ pass:
118
+ task_rate: 0.25 # one <question id>_rate per question; a task's id is "task"
119
+ ```
120
+
121
+ Visitors in task mode really submit forms. Point the engine only at a copy of
122
+ the app that is safe to act on.
123
+
124
+ ### Personas
125
+
126
+ ```yaml
127
+ id: collector
128
+ archetype: A man in his forties who collects sci-fi props
129
+ profile:
130
+ role: IT manager
131
+ experience: Builds replica props, comfortable with electronics kits
132
+ tech_comfort: high
133
+ already_uses: [3D printer, microcontroller kits]
134
+ arrival:
135
+ source: Instagram ad
136
+ saw: "The robot sidekick you wanted as a kid" # all they saw before landing
137
+ needs: [what it looks like, whether it can be modified]
138
+ objections: [AI gadgets oversell, backed projects that never shipped]
139
+ dials: { reading_wpm: 270, skim_rate: 0.5 } # mis_tap_rate is optional, default 0.03
140
+ ```
141
+
142
+ ## The report
143
+
144
+ `reports/<run>.json` has `schema: "sdd-report/1"`.
145
+
146
+ - `verdict`: `pass`, `not met`, `inside the noise`, or `incomplete`.
147
+ - A threshold is met only when the low end of its 95% interval clears it, on
148
+ both the working set and the held-back set. A few visits rarely settle
149
+ anything, and the report says so plainly.
150
+ - `working` has the detail: per question, per persona, what was missed, where
151
+ visitors stopped, and a sample of failing answers. `holdout` has rates only.
152
+ - `costUsd` is the spend at list prices. `blockedRequests` lists outside origins
153
+ that were refused. `error` is present when the engine could not finish.
154
+
155
+ ## Models
156
+
157
+ Chosen by environment variable, with defaults:
158
+
159
+ | Variable | Role | Default |
160
+ |---|---|---|
161
+ | `SDD_MODEL_PERSONA` | the visitors | `claude-haiku-5-5` |
162
+ | `SDD_MODEL_JUDGE` | grades answers and outcomes | `claude-sonnet-5-5` |
163
+
164
+ ## Develop
165
+
166
+ Working on the engine itself needs Node 22.18 or later, because the tests and
167
+ the example app are TypeScript that Node runs directly. The published command
168
+ is compiled and runs on Node 20.11 or later.
169
+
170
+ ```bash
171
+ npm install
172
+ npm run typecheck
173
+ npm test # offline: the model is scripted
174
+ npm run build # compiles to dist/, which is what the published command runs
175
+ ```
176
+
177
+ ## License
178
+
179
+ MIT
@@ -0,0 +1,95 @@
1
+ #!/usr/bin/env node
2
+ // The `sdd` command.
3
+ //
4
+ // sdd simulate ... run simulated visitors against a local app and write a report
5
+ // sdd setup download the browser the engine drives
6
+ // sdd --version
7
+ // sdd --help
8
+ import { spawnSync } from 'node:child_process';
9
+ import { readFile } from 'node:fs/promises';
10
+ import { createRequire } from 'node:module';
11
+ import path from 'node:path';
12
+ import { approveCommand, deriveCommand, designCommand, reviseCommand, scoreCommand, showCommand } from "../pipeline/cli.js";
13
+ import { simulate, USAGE } from "./simulate.js";
14
+ const HELP = `sdd: Simulation Driven Development engine
15
+
16
+ sdd simulate --workdir <dir> --target http://localhost:PORT --run <id> --budget-usd <n>
17
+ [--entry /path] [--runs n] [--concurrency n] [--outside-requests fail|block]
18
+ sdd setup download the browser the engine drives (once, after installing)
19
+ sdd --version
20
+
21
+ sdd derive "<the result you want>" [--source <folder or address>]... [--budget-usd 5]
22
+ turn one sentence into a spec: who it is for, what counts as
23
+ success, thresholds, held-back people, assumptions, and a
24
+ ledger of true claims. Nothing is simulated or built.
25
+ sdd revise <project> "<what to change>" the owner's corrections, applied by the deriver
26
+ sdd approve <project> --budget-usd <n> the owner approves this exact spec
27
+ sdd design <project> [--iterations 6] [--runs 4] [--try 5] design on mocks, score, revise
28
+ sdd score <project> --page <folder> --name <label> score a page the pipeline did not design
29
+ (the site as it stands, say) with the same people and checks
30
+ sdd show [<project>] list projects, or print one project's derived spec
31
+
32
+ The work directory holds spec.yaml, personas/working/*.yaml and
33
+ personas/holdout/*.yaml. The report is written to <workdir>/reports/<run>.json
34
+ and .md. The target must be an app running on this machine; nothing outside it
35
+ is ever reached.
36
+
37
+ Exit codes: 0 report written, 1 runs failed or the engine hit an error (the
38
+ report says which), 2 bad arguments, 3 budget reached.
39
+
40
+ Needs: Node 20.11 or later, the browser from "sdd setup", and the claude CLI
41
+ logged in.`;
42
+ async function version() {
43
+ // package.json sits one level above this file in the source tree and two
44
+ // levels above it once built into dist/engine.
45
+ for (const up of ['../package.json', '../../package.json']) {
46
+ try {
47
+ const pkg = JSON.parse(await readFile(new URL(up, import.meta.url), 'utf8'));
48
+ if (pkg.version)
49
+ return pkg.version;
50
+ }
51
+ catch { /* try the next location */ }
52
+ }
53
+ return 'unknown';
54
+ }
55
+ const [command, ...rest] = process.argv.slice(2);
56
+ if (command === '--version' || command === '-v') {
57
+ console.log(await version());
58
+ }
59
+ else if (command === 'setup') {
60
+ // Download the browser build this engine's own Playwright expects. Running
61
+ // "npx playwright install" from somewhere else can fetch a different build.
62
+ const playwrightCli = path.join(path.dirname(createRequire(import.meta.url).resolve('playwright/package.json')), 'cli.js');
63
+ const result = spawnSync(process.execPath, [playwrightCli, 'install', 'chromium'], { stdio: 'inherit' });
64
+ process.exit(result.status ?? 1);
65
+ }
66
+ else if (command === 'simulate') {
67
+ // exit() and not exitCode: a browser or child process that is slow to close must not hold the command open.
68
+ process.exit(await simulate(rest));
69
+ }
70
+ else if (command === 'derive') {
71
+ process.exit(await deriveCommand(rest));
72
+ }
73
+ else if (command === 'revise') {
74
+ process.exit(await reviseCommand(rest));
75
+ }
76
+ else if (command === 'approve') {
77
+ process.exit(await approveCommand(rest));
78
+ }
79
+ else if (command === 'design') {
80
+ process.exit(await designCommand(rest));
81
+ }
82
+ else if (command === 'score') {
83
+ process.exit(await scoreCommand(rest));
84
+ }
85
+ else if (command === 'show') {
86
+ process.exit(await showCommand(rest));
87
+ }
88
+ else if (command === undefined || command === '--help' || command === '-h' || command === 'help') {
89
+ console.log(HELP);
90
+ }
91
+ else {
92
+ console.error(`Unknown command: ${command}`);
93
+ console.error(USAGE);
94
+ process.exitCode = 2;
95
+ }
@@ -0,0 +1,214 @@
1
+ // The engine's single entry point for a host product such as TurkeyCode.
2
+ //
3
+ // sdd simulate --workdir <dir> --target http://localhost:PORT
4
+ // --run <id> --budget-usd <n>
5
+ // [--entry /path] [--runs n] [--concurrency 6]
6
+ // [--outside-requests fail|block]
7
+ //
8
+ // The work directory holds everything for one app:
9
+ //
10
+ // <workdir>/spec.yaml the simulation spec
11
+ // <workdir>/personas/working/*.yaml personas the builder may see
12
+ // <workdir>/personas/holdout/*.yaml personas held back
13
+ //
14
+ // The engine writes, under the same directory:
15
+ //
16
+ // traces/<experiment>/*.json one per persona run, never edited
17
+ // shots/... screenshots the personas were shown
18
+ // reports/<run>.json the result a product renders
19
+ // reports/<run>.md the same, for a person to read
20
+ //
21
+ // Report only. It measures and reports. It changes nothing in the app.
22
+ // The target must be an app running on this machine. Every request to any
23
+ // other origin is blocked. By default one attempt fails the run. With
24
+ // --outside-requests block the run carries on and the report lists what was
25
+ // refused, for apps that load fonts or scripts from elsewhere.
26
+ //
27
+ // Exit codes: 0 report written, 1 some runs failed or the engine hit an error
28
+ // (a report is written either way; an error gives verdict "incomplete"),
29
+ // 2 bad arguments, 3 budget reached (report covers the runs that finished).
30
+ import { mkdir, readFile, writeFile } from 'node:fs/promises';
31
+ import path from 'node:path';
32
+ import { parseArgs } from 'node:util';
33
+ import { parse } from 'yaml';
34
+ import { runBatch } from "../runner/batch-lib.js";
35
+ import { createLLM } from "../runner/llm/index.js";
36
+ import { loadSpec } from "../runner/spec.js";
37
+ import { summarize } from "../scorer/summary.js";
38
+ export const USAGE = 'Usage: sdd simulate --workdir <dir> --target http://localhost:PORT --run <id> --budget-usd <n> ' +
39
+ '[--entry /path] [--runs n] [--concurrency n] [--outside-requests fail|block]';
40
+ /** Runs one simulation and returns the process exit code. */
41
+ export async function simulate(args, deps = {}) {
42
+ const log = deps.log ?? console.log;
43
+ const logError = deps.logError ?? console.error;
44
+ let values;
45
+ try {
46
+ ({ values } = parseArgs({
47
+ args,
48
+ options: {
49
+ workdir: { type: 'string' },
50
+ target: { type: 'string' },
51
+ entry: { type: 'string', default: '/' },
52
+ run: { type: 'string' },
53
+ runs: { type: 'string' },
54
+ concurrency: { type: 'string', default: '6' },
55
+ 'budget-usd': { type: 'string' },
56
+ 'outside-requests': { type: 'string', default: 'fail' },
57
+ },
58
+ }));
59
+ }
60
+ catch (err) {
61
+ logError(err instanceof Error ? err.message : String(err));
62
+ logError(USAGE);
63
+ return 2;
64
+ }
65
+ if (!values.workdir || !values.target || !values.run || !values['budget-usd']) {
66
+ logError(USAGE);
67
+ return 2;
68
+ }
69
+ const outsideRequests = values['outside-requests'];
70
+ if (outsideRequests !== 'fail' && outsideRequests !== 'block') {
71
+ logError('--outside-requests must be "fail" or "block"');
72
+ return 2;
73
+ }
74
+ const run = values.run;
75
+ const target = `${values.target}${values.entry}`;
76
+ const workDir = path.resolve(values.workdir);
77
+ const outDir = path.join(workDir, 'reports');
78
+ const reportFile = path.join(outDir, `${run}.json`);
79
+ const startedAt = new Date().toISOString();
80
+ try {
81
+ const specFile = path.join(workDir, 'spec.yaml');
82
+ const spec = await loadSpec(specFile);
83
+ // Inside a work directory the persona folders are fixed. The spec cannot point elsewhere.
84
+ spec.personas = { ...spec.personas, working: 'personas/working', holdout: 'personas/holdout' };
85
+ const rawPass = parse(await readFile(specFile, 'utf8')).pass ?? {};
86
+ const thresholds = {};
87
+ for (const q of spec.questions) {
88
+ const t = Number(rawPass[`${q.id}_rate`]);
89
+ if (!(t > 0 && t <= 1))
90
+ throw new Error(`spec.yaml: pass.${q.id}_rate must be a number between 0 and 1`);
91
+ thresholds[q.id] = t;
92
+ }
93
+ const result = await runBatch({
94
+ repoRoot: workDir,
95
+ spec,
96
+ batch: run,
97
+ targetOrigin: values.target,
98
+ entryPath: values.entry,
99
+ sets: ['working', 'holdout'],
100
+ viewports: spec.entry.viewports,
101
+ runs: Number(values.runs ?? spec.personas.runs_per_persona),
102
+ concurrency: Number(values.concurrency),
103
+ budgetUsd: Number(values['budget-usd']),
104
+ outsideRequests,
105
+ llm: deps.llm ?? createLLM(),
106
+ log,
107
+ });
108
+ const working = summarize(spec, thresholds, result.traces, 'working');
109
+ const holdout = summarize(spec, thresholds, result.traces, 'holdout');
110
+ const complete = result.failed === 0 && !result.budgetHit;
111
+ const costUsd = result.traces.reduce((sum, t) => sum + t.cost.usd, 0);
112
+ const calibrated = result.traces.length > 0 && result.traces.every((t) => t.dials.calibrated);
113
+ // "pass" needs every lower bound to clear on both sets. Anything else is
114
+ // reported as it is: not met, or not enough runs to say.
115
+ const verdict = !complete ? 'incomplete'
116
+ : working.met && holdout.met ? 'pass'
117
+ : [...working.questions, ...holdout.questions].some((q) => q.verdict === 'not met') ? 'not met'
118
+ : 'inside the noise';
119
+ /** Holdout detail stays out of anything a builder might read: rates only. */
120
+ const strip = (s) => s.set === 'working'
121
+ ? s
122
+ : { set: s.set, runs: s.runs, questions: s.questions, shortfall: s.shortfall, minMargin: s.minMargin, met: s.met, ratesClear: s.ratesClear };
123
+ const report = {
124
+ schema: 'sdd-report/1',
125
+ run,
126
+ experiment: spec.experiment,
127
+ goal: spec.goal,
128
+ target,
129
+ startedAt,
130
+ finishedAt: new Date().toISOString(),
131
+ verdict,
132
+ calibrated,
133
+ runs: { completed: result.traces.length, failed: result.failed, budgetHit: result.budgetHit },
134
+ costUsd: Math.round(costUsd * 10000) / 10000,
135
+ blockedRequests: result.blocked,
136
+ page: result.traces[0]?.page,
137
+ questions: spec.questions.map((q) => ({ id: q.id, ask: q.ask })),
138
+ working: strip(working),
139
+ holdout: strip(holdout),
140
+ };
141
+ const pct = (x) => (Number.isNaN(x) ? 'n/a' : `${Math.round(x * 100)}%`);
142
+ const table = (s) => ['| Question | Passed | 95% interval | Threshold | Verdict |', '|---|---|---|---|---|',
143
+ ...s.questions.map((q) => `| ${q.id} | ${pct(q.interval.rate)} (${q.interval.passed}/${q.interval.n}) | ${pct(q.interval.lower)} to ${pct(q.interval.upper)} | ${pct(q.threshold)} | ${q.verdict} |`)].join('\n');
144
+ const blockedLine = result.blocked.length === 0
145
+ ? ''
146
+ : `- Outside requests refused, so visitors saw the app without them: ${result.blocked.map((b) => `${b.origin} (${b.count})`).join(', ')}\n`;
147
+ const md = `# Simulation report: ${spec.experiment}, run "${run}"
148
+
149
+ **Verdict: ${verdict}.** A threshold is met only when the low end of its 95% interval clears it, for both the working readers and the held-back readers.
150
+ ${calibrated ? '' : '\n**Not calibrated.** These are simulated visitors that have not been checked against real ones. The scores compare versions of this page with each other. They are not a prediction about real visitors.\n'}
151
+ - Goal: ${spec.goal.trim()}
152
+ - Target: ${target}
153
+ - Runs: ${result.traces.length} completed, ${result.failed} failed${result.budgetHit ? ', stopped by the budget' : ''}
154
+ - Cost at list prices: $${costUsd.toFixed(2)}
155
+ ${blockedLine}
156
+ ## Working readers
157
+
158
+ ${table(working)}
159
+
160
+ ## Held-back readers
161
+
162
+ ${table(holdout)}
163
+
164
+ ## What readers missed most
165
+
166
+ ${working.missedElements.map((m) => `- ${m.question}: "${m.element}" missed by ${pct(m.missedRate)}`).join('\n')}
167
+
168
+ ## Where readers were when time ran out
169
+
170
+ ${Object.entries(working.stoppedIn).map(([name, x]) => `- ${name}: ${pct(x)}`).join('\n')}
171
+
172
+ ## A sample of failing answers
173
+
174
+ ${working.failingAnswers.map((f) => `- [${f.question}] ${f.answer}\n - Judge: ${f.judgeNote}`).join('\n')}
175
+ `;
176
+ await mkdir(outDir, { recursive: true });
177
+ await writeFile(reportFile, JSON.stringify(report, null, 2) + '\n');
178
+ await writeFile(path.join(outDir, `${run}.md`), md);
179
+ log(`Verdict: ${verdict}. Report: ${reportFile}`);
180
+ return result.budgetHit ? 3 : result.failed > 0 ? 1 : 0;
181
+ }
182
+ catch (err) {
183
+ // Exit 1 always leaves a report, so a host never has to guess what happened.
184
+ const raw = err instanceof Error ? err.message : String(err);
185
+ // Playwright's own advice ("npx playwright install") can fetch a build
186
+ // that does not match this engine. Point at the command that does.
187
+ const error = /Executable doesn't exist/.test(raw)
188
+ ? `The browser the engine drives is not installed. Run: sdd setup\n(${raw.split('\n')[0]})`
189
+ : raw;
190
+ const empty = (set) => ({ set, runs: 0, questions: [], met: false });
191
+ const report = {
192
+ schema: 'sdd-report/1',
193
+ run,
194
+ target,
195
+ startedAt,
196
+ finishedAt: new Date().toISOString(),
197
+ verdict: 'incomplete',
198
+ calibrated: false,
199
+ error,
200
+ runs: { completed: 0, failed: 0, budgetHit: false },
201
+ costUsd: 0,
202
+ blockedRequests: [],
203
+ questions: [],
204
+ working: empty('working'),
205
+ holdout: empty('holdout'),
206
+ };
207
+ await mkdir(outDir, { recursive: true });
208
+ await writeFile(reportFile, JSON.stringify(report, null, 2) + '\n');
209
+ await writeFile(path.join(outDir, `${run}.md`), `# Simulation report: run "${run}"\n\n**Verdict: incomplete.** The engine stopped before any result could be scored.\n\n ${error.split('\n').join('\n ')}\n`);
210
+ logError(`Simulation failed: ${error}`);
211
+ logError(`Report: ${reportFile}`);
212
+ return 1;
213
+ }
214
+ }
@@ -0,0 +1,45 @@
1
+ // Judges whether a spoken conversation ended with the expected outcome.
2
+ // Part of the referee.
3
+ //
4
+ // The judge sees the expected outcome and the transcript: what the person
5
+ // said, what the product said back, and what the product did (its events).
6
+ // It does not see the persona, the persona model's reasoning, or whether the
7
+ // person believed it worked.
8
+ const SYSTEM = `You check whether a conversation between a person and a voice assistant ended with the expected outcome. You are strict.
9
+
10
+ You get the expected outcome and the transcript. Each turn has what the person said, what the assistant said back, and a list of what the assistant actually did (tools it ran). Decide whether the transcript shows the expected outcome.
11
+
12
+ Rules:
13
+ - Evidence first. Quote the exact words or name the exact action in the transcript that show the outcome. If there are none, write "none". Decide only after quoting.
14
+ - Saying is not doing. If the outcome needs an action and the assistant only said it did it, with no matching entry under what it did, the outcome is not met.
15
+ - A promise to do it later, a question back to the person, or an offer is not the outcome.
16
+ - A wrong name, wrong amount, or wrong thing is not the outcome, even if the assistant sounds confident.
17
+ - Do not give credit for what probably happened. Judge only what the transcript shows.
18
+ - The note is one short sentence: what shows the outcome, or what is missing.`;
19
+ export function buildConversationRequest(expected, when, turns) {
20
+ const transcript = turns
21
+ .map((t, i) => `Turn ${i + 1}\n Person: "${t.said}"\n Assistant: ${t.heard ? `"${t.heard}"` : '(said nothing)'}\n Assistant did: ${t.events.length ? t.events.map((e) => (e.detail ? `${e.name} ${e.detail}` : e.name)).join('; ') : '(nothing)'}`)
22
+ .join('\n');
23
+ return {
24
+ role: 'judge',
25
+ system: SYSTEM,
26
+ prompt: `EXPECTED OUTCOME\n${expected}\n\nWHEN\nThe conversation took place on ${when}, local time. Times in the assistant's actions may be written in UTC.\n\nTRANSCRIPT\n${transcript || '(no turns)'}\n\nDoes the transcript show the expected outcome?`,
27
+ schema: {
28
+ type: 'object',
29
+ properties: { evidence: { type: 'string' }, met: { type: 'boolean' }, note: { type: 'string' } },
30
+ required: ['evidence', 'met', 'note'],
31
+ },
32
+ };
33
+ }
34
+ export async function judgeConversation(llm, expected, when, turns) {
35
+ const request = buildConversationRequest(expected, when, turns);
36
+ const calls = [];
37
+ for (let attempt = 0; attempt < 2; attempt++) {
38
+ const response = await llm.complete(request);
39
+ calls.push({ request, response });
40
+ const reply = response.structured;
41
+ if (reply && typeof reply.met === 'boolean')
42
+ return { met: reply.met, evidence: reply.evidence ?? '', note: reply.note ?? '', calls };
43
+ }
44
+ throw new Error('The conversation judge gave no verdict');
45
+ }