sdd-engine 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +178 -2
- package/dist/engine/cli.js +95 -0
- package/dist/engine/simulate.js +214 -0
- package/dist/judge/conversation.js +45 -0
- package/dist/judge/judge.js +122 -0
- package/dist/judge/outcome.js +42 -0
- package/dist/judge/regrade.js +74 -0
- package/dist/pipeline/agent.js +78 -0
- package/dist/pipeline/agentstage.js +88 -0
- package/dist/pipeline/approve.js +68 -0
- package/dist/pipeline/cli.js +261 -0
- package/dist/pipeline/derive.js +287 -0
- package/dist/pipeline/designloop.js +385 -0
- package/dist/pipeline/facts.js +108 -0
- package/dist/pipeline/mock.js +253 -0
- package/dist/pipeline/project.js +79 -0
- package/dist/pipeline/render.js +95 -0
- package/dist/pipeline/spec.js +272 -0
- package/dist/pipeline/try.js +162 -0
- package/dist/runner/batch-lib.js +157 -0
- package/dist/runner/batch.js +56 -0
- package/dist/runner/cli.js +61 -0
- package/dist/runner/llm/claude-code.js +123 -0
- package/dist/runner/llm/index.js +12 -0
- package/dist/runner/llm/types.js +17 -0
- package/dist/runner/reader/answer.js +76 -0
- package/dist/runner/reader/extract.js +106 -0
- package/dist/runner/reader/run.js +111 -0
- package/dist/runner/reader/skim.js +115 -0
- package/dist/runner/rng.js +22 -0
- package/dist/runner/sandbox/cli.js +60 -0
- package/dist/runner/sandbox/sandbox.js +144 -0
- package/dist/runner/sandbox/server.js +64 -0
- package/dist/runner/spec.js +84 -0
- package/dist/runner/talker/decide.js +110 -0
- package/dist/runner/talker/run.js +159 -0
- package/dist/runner/talker/world.js +36 -0
- package/dist/runner/trace.js +130 -0
- package/dist/runner/visitor/decide.js +159 -0
- package/dist/runner/visitor/observe.js +102 -0
- package/dist/runner/visitor/run.js +296 -0
- package/dist/scorer/compare.js +119 -0
- package/dist/scorer/score.js +201 -0
- package/dist/scorer/spot-check-compare.js +41 -0
- package/dist/scorer/spot-check-view.js +45 -0
- package/dist/scorer/stats.js +45 -0
- package/dist/scorer/summary.js +62 -0
- package/package.json +49 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Chad Cox (rangerchaz)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,179 @@
|
|
|
1
|
-
#
|
|
1
|
+
# sdd-engine
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
The engine for Simulation Driven Development (SDD).
|
|
4
|
+
|
|
5
|
+
A test asks "is the code correct?" and answers pass or fail. A simulation asks
|
|
6
|
+
"does this work for the people it is for?" and answers with a score against a
|
|
7
|
+
threshold. This engine sends simulated visitors through an app running on your
|
|
8
|
+
machine, scores what happened, and writes a report.
|
|
9
|
+
|
|
10
|
+
It only measures. It never changes the app.
|
|
11
|
+
|
|
12
|
+
**The scores are not calibrated.** The simulated visitors have not been checked
|
|
13
|
+
against real people. A score lets you compare two versions of the same app. It
|
|
14
|
+
is not a prediction of what real visitors will do.
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
npm install -g sdd-engine
|
|
20
|
+
sdd setup # downloads the browser the engine drives, once
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
If the global install fails with `EACCES`, your user cannot write to npm's
|
|
24
|
+
global folder. That is the default on Linux with the system Node and on the
|
|
25
|
+
official Docker image. Run it with `npx` instead, which needs no install:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
npx sdd-engine setup
|
|
29
|
+
npx sdd-engine --help
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Needs Node 20.11 or later, and the [Claude Code](https://claude.com/claude-code)
|
|
33
|
+
CLI installed and logged in. The engine calls models through `claude -p`, so a
|
|
34
|
+
run is billed to whatever that CLI is logged in with.
|
|
35
|
+
|
|
36
|
+
## Try it
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
git clone https://github.com/TurkeyCode-ai/sdd && cd sdd
|
|
40
|
+
node examples/waitlist/app.ts & # a stand-in app on http://127.0.0.1:4180
|
|
41
|
+
|
|
42
|
+
sdd simulate --workdir examples/waitlist --target http://127.0.0.1:4180 \
|
|
43
|
+
--run demo --runs 2 --budget-usd 1
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
The report lands in `examples/waitlist/reports/demo.md` and `demo.json`.
|
|
47
|
+
|
|
48
|
+
## The command
|
|
49
|
+
|
|
50
|
+
```
|
|
51
|
+
sdd simulate --workdir <dir> --target http://localhost:PORT --run <id> --budget-usd <n>
|
|
52
|
+
[--entry /path] [--runs n] [--concurrency n] [--outside-requests fail|block]
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
| Flag | Meaning |
|
|
56
|
+
|---|---|
|
|
57
|
+
| `--workdir` | The work directory (below). Traces, screenshots and reports are written into it. |
|
|
58
|
+
| `--target` | The app to visit. It must be on this machine: `localhost`, `127.0.0.1` or `[::1]`. |
|
|
59
|
+
| `--run` | A name for this run. Using the same name again resumes it: finished visits are skipped. |
|
|
60
|
+
| `--budget-usd` | Stop starting new visits once this much is spent, at list prices. |
|
|
61
|
+
| `--entry` | The path visitors land on. Default `/`. |
|
|
62
|
+
| `--runs` | Visits per persona and viewport. Default: `personas.runs_per_persona` in the spec. |
|
|
63
|
+
| `--concurrency` | Visits at once. Default 6. |
|
|
64
|
+
| `--outside-requests` | `fail` (default): one request to another origin fails the run. `block`: such requests are refused and listed in the report, and the run carries on. Use `block` for apps that load fonts or scripts from elsewhere. Either way nothing outside the target is ever reached. |
|
|
65
|
+
|
|
66
|
+
Exit codes: `0` report written. `1` some visits failed, or the engine hit an
|
|
67
|
+
error; a report is written either way and says which. `2` bad arguments. `3`
|
|
68
|
+
budget reached; the report covers the visits that finished.
|
|
69
|
+
|
|
70
|
+
## The work directory
|
|
71
|
+
|
|
72
|
+
```
|
|
73
|
+
<workdir>/
|
|
74
|
+
spec.yaml what is simulated and what counts as passing
|
|
75
|
+
personas/working/*.yaml visitors whose results are reported in full
|
|
76
|
+
personas/holdout/*.yaml visitors held back; only their pass rates are reported
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
The held-back set exists so that whoever revises the app cannot tune it to the
|
|
80
|
+
visitors they can see. Keep it away from them.
|
|
81
|
+
|
|
82
|
+
### spec.yaml
|
|
83
|
+
|
|
84
|
+
Two kinds. **Reader mode** (`kind: comprehension`): visitors skim the entry page
|
|
85
|
+
for a fixed time, the page is taken away, and they answer questions from what
|
|
86
|
+
they actually took in. A separate judge grades each answer against a rubric.
|
|
87
|
+
See `examples/sdd-site/spec.yaml`.
|
|
88
|
+
|
|
89
|
+
**Visitor mode** (`kind: task`): visitors act on the app. They can scroll, click,
|
|
90
|
+
type, or leave. Leaving is the default: any other action has to quote words that
|
|
91
|
+
are really on the screen. The model is never told what success is. The engine
|
|
92
|
+
checks it from what the browser did. See `examples/waitlist/spec.yaml`.
|
|
93
|
+
|
|
94
|
+
```yaml
|
|
95
|
+
experiment: waitlist-example
|
|
96
|
+
kind: task
|
|
97
|
+
goal: A visitor lands from an ad and joins the waitlist.
|
|
98
|
+
entry:
|
|
99
|
+
from: a social media ad
|
|
100
|
+
viewports: [mobile] # mobile, desktop
|
|
101
|
+
task:
|
|
102
|
+
goal: Join the waitlist
|
|
103
|
+
instruction: ... # optional: what the visitor was asked to try
|
|
104
|
+
success: # any of these; never shown to the model
|
|
105
|
+
request: { method: POST, path_includes: /api/waitlist }
|
|
106
|
+
url_includes: /thanks
|
|
107
|
+
text_visible: You are on the list
|
|
108
|
+
leaves_to: [github.com/acme/widget] # follows an outside link to here; the request is never made
|
|
109
|
+
clicks: [Copy] # presses a control whose label includes this
|
|
110
|
+
outcome: A confirmation that the visitor is on the list # judged from the final screen
|
|
111
|
+
visitor:
|
|
112
|
+
patience_seconds: 75
|
|
113
|
+
max_steps: 15
|
|
114
|
+
screenshots: true
|
|
115
|
+
personas:
|
|
116
|
+
runs_per_persona: 10
|
|
117
|
+
pass:
|
|
118
|
+
task_rate: 0.25 # one <question id>_rate per question; a task's id is "task"
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Visitors in task mode really submit forms. Point the engine only at a copy of
|
|
122
|
+
the app that is safe to act on.
|
|
123
|
+
|
|
124
|
+
### Personas
|
|
125
|
+
|
|
126
|
+
```yaml
|
|
127
|
+
id: collector
|
|
128
|
+
archetype: A man in his forties who collects sci-fi props
|
|
129
|
+
profile:
|
|
130
|
+
role: IT manager
|
|
131
|
+
experience: Builds replica props, comfortable with electronics kits
|
|
132
|
+
tech_comfort: high
|
|
133
|
+
already_uses: [3D printer, microcontroller kits]
|
|
134
|
+
arrival:
|
|
135
|
+
source: Instagram ad
|
|
136
|
+
saw: "The robot sidekick you wanted as a kid" # all they saw before landing
|
|
137
|
+
needs: [what it looks like, whether it can be modified]
|
|
138
|
+
objections: [AI gadgets oversell, backed projects that never shipped]
|
|
139
|
+
dials: { reading_wpm: 270, skim_rate: 0.5 } # mis_tap_rate is optional, default 0.03
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## The report
|
|
143
|
+
|
|
144
|
+
`reports/<run>.json` has `schema: "sdd-report/1"`.
|
|
145
|
+
|
|
146
|
+
- `verdict`: `pass`, `not met`, `inside the noise`, or `incomplete`.
|
|
147
|
+
- A threshold is met only when the low end of its 95% interval clears it, on
|
|
148
|
+
both the working set and the held-back set. A few visits rarely settle
|
|
149
|
+
anything, and the report says so plainly.
|
|
150
|
+
- `working` has the detail: per question, per persona, what was missed, where
|
|
151
|
+
visitors stopped, and a sample of failing answers. `holdout` has rates only.
|
|
152
|
+
- `costUsd` is the spend at list prices. `blockedRequests` lists outside origins
|
|
153
|
+
that were refused. `error` is present when the engine could not finish.
|
|
154
|
+
|
|
155
|
+
## Models
|
|
156
|
+
|
|
157
|
+
Chosen by environment variable, with defaults:
|
|
158
|
+
|
|
159
|
+
| Variable | Role | Default |
|
|
160
|
+
|---|---|---|
|
|
161
|
+
| `SDD_MODEL_PERSONA` | the visitors | `claude-haiku-5-5` |
|
|
162
|
+
| `SDD_MODEL_JUDGE` | grades answers and outcomes | `claude-sonnet-5-5` |
|
|
163
|
+
|
|
164
|
+
## Develop
|
|
165
|
+
|
|
166
|
+
Working on the engine itself needs Node 22.18 or later, because the tests and
|
|
167
|
+
the example app are TypeScript that Node runs directly. The published command
|
|
168
|
+
is compiled and runs on Node 20.11 or later.
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
npm install
|
|
172
|
+
npm run typecheck
|
|
173
|
+
npm test # offline: the model is scripted
|
|
174
|
+
npm run build # compiles to dist/, which is what the published command runs
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
## License
|
|
178
|
+
|
|
179
|
+
MIT
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// The `sdd` command.
|
|
3
|
+
//
|
|
4
|
+
// sdd simulate ... run simulated visitors against a local app and write a report
|
|
5
|
+
// sdd setup download the browser the engine drives
|
|
6
|
+
// sdd --version
|
|
7
|
+
// sdd --help
|
|
8
|
+
import { spawnSync } from 'node:child_process';
|
|
9
|
+
import { readFile } from 'node:fs/promises';
|
|
10
|
+
import { createRequire } from 'node:module';
|
|
11
|
+
import path from 'node:path';
|
|
12
|
+
import { approveCommand, deriveCommand, designCommand, reviseCommand, scoreCommand, showCommand } from "../pipeline/cli.js";
|
|
13
|
+
import { simulate, USAGE } from "./simulate.js";
|
|
14
|
+
const HELP = `sdd: Simulation Driven Development engine
|
|
15
|
+
|
|
16
|
+
sdd simulate --workdir <dir> --target http://localhost:PORT --run <id> --budget-usd <n>
|
|
17
|
+
[--entry /path] [--runs n] [--concurrency n] [--outside-requests fail|block]
|
|
18
|
+
sdd setup download the browser the engine drives (once, after installing)
|
|
19
|
+
sdd --version
|
|
20
|
+
|
|
21
|
+
sdd derive "<the result you want>" [--source <folder or address>]... [--budget-usd 5]
|
|
22
|
+
turn one sentence into a spec: who it is for, what counts as
|
|
23
|
+
success, thresholds, held-back people, assumptions, and a
|
|
24
|
+
ledger of true claims. Nothing is simulated or built.
|
|
25
|
+
sdd revise <project> "<what to change>" the owner's corrections, applied by the deriver
|
|
26
|
+
sdd approve <project> --budget-usd <n> the owner approves this exact spec
|
|
27
|
+
sdd design <project> [--iterations 6] [--runs 4] [--try 5] design on mocks, score, revise
|
|
28
|
+
sdd score <project> --page <folder> --name <label> score a page the pipeline did not design
|
|
29
|
+
(the site as it stands, say) with the same people and checks
|
|
30
|
+
sdd show [<project>] list projects, or print one project's derived spec
|
|
31
|
+
|
|
32
|
+
The work directory holds spec.yaml, personas/working/*.yaml and
|
|
33
|
+
personas/holdout/*.yaml. The report is written to <workdir>/reports/<run>.json
|
|
34
|
+
and .md. The target must be an app running on this machine; nothing outside it
|
|
35
|
+
is ever reached.
|
|
36
|
+
|
|
37
|
+
Exit codes: 0 report written, 1 runs failed or the engine hit an error (the
|
|
38
|
+
report says which), 2 bad arguments, 3 budget reached.
|
|
39
|
+
|
|
40
|
+
Needs: Node 20.11 or later, the browser from "sdd setup", and the claude CLI
|
|
41
|
+
logged in.`;
|
|
42
|
+
async function version() {
|
|
43
|
+
// package.json sits one level above this file in the source tree and two
|
|
44
|
+
// levels above it once built into dist/engine.
|
|
45
|
+
for (const up of ['../package.json', '../../package.json']) {
|
|
46
|
+
try {
|
|
47
|
+
const pkg = JSON.parse(await readFile(new URL(up, import.meta.url), 'utf8'));
|
|
48
|
+
if (pkg.version)
|
|
49
|
+
return pkg.version;
|
|
50
|
+
}
|
|
51
|
+
catch { /* try the next location */ }
|
|
52
|
+
}
|
|
53
|
+
return 'unknown';
|
|
54
|
+
}
|
|
55
|
+
const [command, ...rest] = process.argv.slice(2);
|
|
56
|
+
if (command === '--version' || command === '-v') {
|
|
57
|
+
console.log(await version());
|
|
58
|
+
}
|
|
59
|
+
else if (command === 'setup') {
|
|
60
|
+
// Download the browser build this engine's own Playwright expects. Running
|
|
61
|
+
// "npx playwright install" from somewhere else can fetch a different build.
|
|
62
|
+
const playwrightCli = path.join(path.dirname(createRequire(import.meta.url).resolve('playwright/package.json')), 'cli.js');
|
|
63
|
+
const result = spawnSync(process.execPath, [playwrightCli, 'install', 'chromium'], { stdio: 'inherit' });
|
|
64
|
+
process.exit(result.status ?? 1);
|
|
65
|
+
}
|
|
66
|
+
else if (command === 'simulate') {
|
|
67
|
+
// exit() and not exitCode: a browser or child process that is slow to close must not hold the command open.
|
|
68
|
+
process.exit(await simulate(rest));
|
|
69
|
+
}
|
|
70
|
+
else if (command === 'derive') {
|
|
71
|
+
process.exit(await deriveCommand(rest));
|
|
72
|
+
}
|
|
73
|
+
else if (command === 'revise') {
|
|
74
|
+
process.exit(await reviseCommand(rest));
|
|
75
|
+
}
|
|
76
|
+
else if (command === 'approve') {
|
|
77
|
+
process.exit(await approveCommand(rest));
|
|
78
|
+
}
|
|
79
|
+
else if (command === 'design') {
|
|
80
|
+
process.exit(await designCommand(rest));
|
|
81
|
+
}
|
|
82
|
+
else if (command === 'score') {
|
|
83
|
+
process.exit(await scoreCommand(rest));
|
|
84
|
+
}
|
|
85
|
+
else if (command === 'show') {
|
|
86
|
+
process.exit(await showCommand(rest));
|
|
87
|
+
}
|
|
88
|
+
else if (command === undefined || command === '--help' || command === '-h' || command === 'help') {
|
|
89
|
+
console.log(HELP);
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
console.error(`Unknown command: ${command}`);
|
|
93
|
+
console.error(USAGE);
|
|
94
|
+
process.exitCode = 2;
|
|
95
|
+
}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
// The engine's single entry point for a host product such as TurkeyCode.
|
|
2
|
+
//
|
|
3
|
+
// sdd simulate --workdir <dir> --target http://localhost:PORT
|
|
4
|
+
// --run <id> --budget-usd <n>
|
|
5
|
+
// [--entry /path] [--runs n] [--concurrency 6]
|
|
6
|
+
// [--outside-requests fail|block]
|
|
7
|
+
//
|
|
8
|
+
// The work directory holds everything for one app:
|
|
9
|
+
//
|
|
10
|
+
// <workdir>/spec.yaml the simulation spec
|
|
11
|
+
// <workdir>/personas/working/*.yaml personas the builder may see
|
|
12
|
+
// <workdir>/personas/holdout/*.yaml personas held back
|
|
13
|
+
//
|
|
14
|
+
// The engine writes, under the same directory:
|
|
15
|
+
//
|
|
16
|
+
// traces/<experiment>/*.json one per persona run, never edited
|
|
17
|
+
// shots/... screenshots the personas were shown
|
|
18
|
+
// reports/<run>.json the result a product renders
|
|
19
|
+
// reports/<run>.md the same, for a person to read
|
|
20
|
+
//
|
|
21
|
+
// Report only. It measures and reports. It changes nothing in the app.
|
|
22
|
+
// The target must be an app running on this machine. Every request to any
|
|
23
|
+
// other origin is blocked. By default one attempt fails the run. With
|
|
24
|
+
// --outside-requests block the run carries on and the report lists what was
|
|
25
|
+
// refused, for apps that load fonts or scripts from elsewhere.
|
|
26
|
+
//
|
|
27
|
+
// Exit codes: 0 report written, 1 some runs failed or the engine hit an error
|
|
28
|
+
// (a report is written either way; an error gives verdict "incomplete"),
|
|
29
|
+
// 2 bad arguments, 3 budget reached (report covers the runs that finished).
|
|
30
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
31
|
+
import path from 'node:path';
|
|
32
|
+
import { parseArgs } from 'node:util';
|
|
33
|
+
import { parse } from 'yaml';
|
|
34
|
+
import { runBatch } from "../runner/batch-lib.js";
|
|
35
|
+
import { createLLM } from "../runner/llm/index.js";
|
|
36
|
+
import { loadSpec } from "../runner/spec.js";
|
|
37
|
+
import { summarize } from "../scorer/summary.js";
|
|
38
|
+
export const USAGE = 'Usage: sdd simulate --workdir <dir> --target http://localhost:PORT --run <id> --budget-usd <n> ' +
|
|
39
|
+
'[--entry /path] [--runs n] [--concurrency n] [--outside-requests fail|block]';
|
|
40
|
+
/** Runs one simulation and returns the process exit code. */
|
|
41
|
+
export async function simulate(args, deps = {}) {
|
|
42
|
+
const log = deps.log ?? console.log;
|
|
43
|
+
const logError = deps.logError ?? console.error;
|
|
44
|
+
let values;
|
|
45
|
+
try {
|
|
46
|
+
({ values } = parseArgs({
|
|
47
|
+
args,
|
|
48
|
+
options: {
|
|
49
|
+
workdir: { type: 'string' },
|
|
50
|
+
target: { type: 'string' },
|
|
51
|
+
entry: { type: 'string', default: '/' },
|
|
52
|
+
run: { type: 'string' },
|
|
53
|
+
runs: { type: 'string' },
|
|
54
|
+
concurrency: { type: 'string', default: '6' },
|
|
55
|
+
'budget-usd': { type: 'string' },
|
|
56
|
+
'outside-requests': { type: 'string', default: 'fail' },
|
|
57
|
+
},
|
|
58
|
+
}));
|
|
59
|
+
}
|
|
60
|
+
catch (err) {
|
|
61
|
+
logError(err instanceof Error ? err.message : String(err));
|
|
62
|
+
logError(USAGE);
|
|
63
|
+
return 2;
|
|
64
|
+
}
|
|
65
|
+
if (!values.workdir || !values.target || !values.run || !values['budget-usd']) {
|
|
66
|
+
logError(USAGE);
|
|
67
|
+
return 2;
|
|
68
|
+
}
|
|
69
|
+
const outsideRequests = values['outside-requests'];
|
|
70
|
+
if (outsideRequests !== 'fail' && outsideRequests !== 'block') {
|
|
71
|
+
logError('--outside-requests must be "fail" or "block"');
|
|
72
|
+
return 2;
|
|
73
|
+
}
|
|
74
|
+
const run = values.run;
|
|
75
|
+
const target = `${values.target}${values.entry}`;
|
|
76
|
+
const workDir = path.resolve(values.workdir);
|
|
77
|
+
const outDir = path.join(workDir, 'reports');
|
|
78
|
+
const reportFile = path.join(outDir, `${run}.json`);
|
|
79
|
+
const startedAt = new Date().toISOString();
|
|
80
|
+
try {
|
|
81
|
+
const specFile = path.join(workDir, 'spec.yaml');
|
|
82
|
+
const spec = await loadSpec(specFile);
|
|
83
|
+
// Inside a work directory the persona folders are fixed. The spec cannot point elsewhere.
|
|
84
|
+
spec.personas = { ...spec.personas, working: 'personas/working', holdout: 'personas/holdout' };
|
|
85
|
+
const rawPass = parse(await readFile(specFile, 'utf8')).pass ?? {};
|
|
86
|
+
const thresholds = {};
|
|
87
|
+
for (const q of spec.questions) {
|
|
88
|
+
const t = Number(rawPass[`${q.id}_rate`]);
|
|
89
|
+
if (!(t > 0 && t <= 1))
|
|
90
|
+
throw new Error(`spec.yaml: pass.${q.id}_rate must be a number between 0 and 1`);
|
|
91
|
+
thresholds[q.id] = t;
|
|
92
|
+
}
|
|
93
|
+
const result = await runBatch({
|
|
94
|
+
repoRoot: workDir,
|
|
95
|
+
spec,
|
|
96
|
+
batch: run,
|
|
97
|
+
targetOrigin: values.target,
|
|
98
|
+
entryPath: values.entry,
|
|
99
|
+
sets: ['working', 'holdout'],
|
|
100
|
+
viewports: spec.entry.viewports,
|
|
101
|
+
runs: Number(values.runs ?? spec.personas.runs_per_persona),
|
|
102
|
+
concurrency: Number(values.concurrency),
|
|
103
|
+
budgetUsd: Number(values['budget-usd']),
|
|
104
|
+
outsideRequests,
|
|
105
|
+
llm: deps.llm ?? createLLM(),
|
|
106
|
+
log,
|
|
107
|
+
});
|
|
108
|
+
const working = summarize(spec, thresholds, result.traces, 'working');
|
|
109
|
+
const holdout = summarize(spec, thresholds, result.traces, 'holdout');
|
|
110
|
+
const complete = result.failed === 0 && !result.budgetHit;
|
|
111
|
+
const costUsd = result.traces.reduce((sum, t) => sum + t.cost.usd, 0);
|
|
112
|
+
const calibrated = result.traces.length > 0 && result.traces.every((t) => t.dials.calibrated);
|
|
113
|
+
// "pass" needs every lower bound to clear on both sets. Anything else is
|
|
114
|
+
// reported as it is: not met, or not enough runs to say.
|
|
115
|
+
const verdict = !complete ? 'incomplete'
|
|
116
|
+
: working.met && holdout.met ? 'pass'
|
|
117
|
+
: [...working.questions, ...holdout.questions].some((q) => q.verdict === 'not met') ? 'not met'
|
|
118
|
+
: 'inside the noise';
|
|
119
|
+
/** Holdout detail stays out of anything a builder might read: rates only. */
|
|
120
|
+
const strip = (s) => s.set === 'working'
|
|
121
|
+
? s
|
|
122
|
+
: { set: s.set, runs: s.runs, questions: s.questions, shortfall: s.shortfall, minMargin: s.minMargin, met: s.met, ratesClear: s.ratesClear };
|
|
123
|
+
const report = {
|
|
124
|
+
schema: 'sdd-report/1',
|
|
125
|
+
run,
|
|
126
|
+
experiment: spec.experiment,
|
|
127
|
+
goal: spec.goal,
|
|
128
|
+
target,
|
|
129
|
+
startedAt,
|
|
130
|
+
finishedAt: new Date().toISOString(),
|
|
131
|
+
verdict,
|
|
132
|
+
calibrated,
|
|
133
|
+
runs: { completed: result.traces.length, failed: result.failed, budgetHit: result.budgetHit },
|
|
134
|
+
costUsd: Math.round(costUsd * 10000) / 10000,
|
|
135
|
+
blockedRequests: result.blocked,
|
|
136
|
+
page: result.traces[0]?.page,
|
|
137
|
+
questions: spec.questions.map((q) => ({ id: q.id, ask: q.ask })),
|
|
138
|
+
working: strip(working),
|
|
139
|
+
holdout: strip(holdout),
|
|
140
|
+
};
|
|
141
|
+
const pct = (x) => (Number.isNaN(x) ? 'n/a' : `${Math.round(x * 100)}%`);
|
|
142
|
+
const table = (s) => ['| Question | Passed | 95% interval | Threshold | Verdict |', '|---|---|---|---|---|',
|
|
143
|
+
...s.questions.map((q) => `| ${q.id} | ${pct(q.interval.rate)} (${q.interval.passed}/${q.interval.n}) | ${pct(q.interval.lower)} to ${pct(q.interval.upper)} | ${pct(q.threshold)} | ${q.verdict} |`)].join('\n');
|
|
144
|
+
const blockedLine = result.blocked.length === 0
|
|
145
|
+
? ''
|
|
146
|
+
: `- Outside requests refused, so visitors saw the app without them: ${result.blocked.map((b) => `${b.origin} (${b.count})`).join(', ')}\n`;
|
|
147
|
+
const md = `# Simulation report: ${spec.experiment}, run "${run}"
|
|
148
|
+
|
|
149
|
+
**Verdict: ${verdict}.** A threshold is met only when the low end of its 95% interval clears it, for both the working readers and the held-back readers.
|
|
150
|
+
${calibrated ? '' : '\n**Not calibrated.** These are simulated visitors that have not been checked against real ones. The scores compare versions of this page with each other. They are not a prediction about real visitors.\n'}
|
|
151
|
+
- Goal: ${spec.goal.trim()}
|
|
152
|
+
- Target: ${target}
|
|
153
|
+
- Runs: ${result.traces.length} completed, ${result.failed} failed${result.budgetHit ? ', stopped by the budget' : ''}
|
|
154
|
+
- Cost at list prices: $${costUsd.toFixed(2)}
|
|
155
|
+
${blockedLine}
|
|
156
|
+
## Working readers
|
|
157
|
+
|
|
158
|
+
${table(working)}
|
|
159
|
+
|
|
160
|
+
## Held-back readers
|
|
161
|
+
|
|
162
|
+
${table(holdout)}
|
|
163
|
+
|
|
164
|
+
## What readers missed most
|
|
165
|
+
|
|
166
|
+
${working.missedElements.map((m) => `- ${m.question}: "${m.element}" missed by ${pct(m.missedRate)}`).join('\n')}
|
|
167
|
+
|
|
168
|
+
## Where readers were when time ran out
|
|
169
|
+
|
|
170
|
+
${Object.entries(working.stoppedIn).map(([name, x]) => `- ${name}: ${pct(x)}`).join('\n')}
|
|
171
|
+
|
|
172
|
+
## A sample of failing answers
|
|
173
|
+
|
|
174
|
+
${working.failingAnswers.map((f) => `- [${f.question}] ${f.answer}\n - Judge: ${f.judgeNote}`).join('\n')}
|
|
175
|
+
`;
|
|
176
|
+
await mkdir(outDir, { recursive: true });
|
|
177
|
+
await writeFile(reportFile, JSON.stringify(report, null, 2) + '\n');
|
|
178
|
+
await writeFile(path.join(outDir, `${run}.md`), md);
|
|
179
|
+
log(`Verdict: ${verdict}. Report: ${reportFile}`);
|
|
180
|
+
return result.budgetHit ? 3 : result.failed > 0 ? 1 : 0;
|
|
181
|
+
}
|
|
182
|
+
catch (err) {
|
|
183
|
+
// Exit 1 always leaves a report, so a host never has to guess what happened.
|
|
184
|
+
const raw = err instanceof Error ? err.message : String(err);
|
|
185
|
+
// Playwright's own advice ("npx playwright install") can fetch a build
|
|
186
|
+
// that does not match this engine. Point at the command that does.
|
|
187
|
+
const error = /Executable doesn't exist/.test(raw)
|
|
188
|
+
? `The browser the engine drives is not installed. Run: sdd setup\n(${raw.split('\n')[0]})`
|
|
189
|
+
: raw;
|
|
190
|
+
const empty = (set) => ({ set, runs: 0, questions: [], met: false });
|
|
191
|
+
const report = {
|
|
192
|
+
schema: 'sdd-report/1',
|
|
193
|
+
run,
|
|
194
|
+
target,
|
|
195
|
+
startedAt,
|
|
196
|
+
finishedAt: new Date().toISOString(),
|
|
197
|
+
verdict: 'incomplete',
|
|
198
|
+
calibrated: false,
|
|
199
|
+
error,
|
|
200
|
+
runs: { completed: 0, failed: 0, budgetHit: false },
|
|
201
|
+
costUsd: 0,
|
|
202
|
+
blockedRequests: [],
|
|
203
|
+
questions: [],
|
|
204
|
+
working: empty('working'),
|
|
205
|
+
holdout: empty('holdout'),
|
|
206
|
+
};
|
|
207
|
+
await mkdir(outDir, { recursive: true });
|
|
208
|
+
await writeFile(reportFile, JSON.stringify(report, null, 2) + '\n');
|
|
209
|
+
await writeFile(path.join(outDir, `${run}.md`), `# Simulation report: run "${run}"\n\n**Verdict: incomplete.** The engine stopped before any result could be scored.\n\n ${error.split('\n').join('\n ')}\n`);
|
|
210
|
+
logError(`Simulation failed: ${error}`);
|
|
211
|
+
logError(`Report: ${reportFile}`);
|
|
212
|
+
return 1;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// Judges whether a spoken conversation ended with the expected outcome.
|
|
2
|
+
// Part of the referee.
|
|
3
|
+
//
|
|
4
|
+
// The judge sees the expected outcome and the transcript: what the person
|
|
5
|
+
// said, what the product said back, and what the product did (its events).
|
|
6
|
+
// It does not see the persona, the persona model's reasoning, or whether the
|
|
7
|
+
// person believed it worked.
|
|
8
|
+
const SYSTEM = `You check whether a conversation between a person and a voice assistant ended with the expected outcome. You are strict.
|
|
9
|
+
|
|
10
|
+
You get the expected outcome and the transcript. Each turn has what the person said, what the assistant said back, and a list of what the assistant actually did (tools it ran). Decide whether the transcript shows the expected outcome.
|
|
11
|
+
|
|
12
|
+
Rules:
|
|
13
|
+
- Evidence first. Quote the exact words or name the exact action in the transcript that show the outcome. If there are none, write "none". Decide only after quoting.
|
|
14
|
+
- Saying is not doing. If the outcome needs an action and the assistant only said it did it, with no matching entry under what it did, the outcome is not met.
|
|
15
|
+
- A promise to do it later, a question back to the person, or an offer is not the outcome.
|
|
16
|
+
- A wrong name, wrong amount, or wrong thing is not the outcome, even if the assistant sounds confident.
|
|
17
|
+
- Do not give credit for what probably happened. Judge only what the transcript shows.
|
|
18
|
+
- The note is one short sentence: what shows the outcome, or what is missing.`;
|
|
19
|
+
export function buildConversationRequest(expected, when, turns) {
|
|
20
|
+
const transcript = turns
|
|
21
|
+
.map((t, i) => `Turn ${i + 1}\n Person: "${t.said}"\n Assistant: ${t.heard ? `"${t.heard}"` : '(said nothing)'}\n Assistant did: ${t.events.length ? t.events.map((e) => (e.detail ? `${e.name} ${e.detail}` : e.name)).join('; ') : '(nothing)'}`)
|
|
22
|
+
.join('\n');
|
|
23
|
+
return {
|
|
24
|
+
role: 'judge',
|
|
25
|
+
system: SYSTEM,
|
|
26
|
+
prompt: `EXPECTED OUTCOME\n${expected}\n\nWHEN\nThe conversation took place on ${when}, local time. Times in the assistant's actions may be written in UTC.\n\nTRANSCRIPT\n${transcript || '(no turns)'}\n\nDoes the transcript show the expected outcome?`,
|
|
27
|
+
schema: {
|
|
28
|
+
type: 'object',
|
|
29
|
+
properties: { evidence: { type: 'string' }, met: { type: 'boolean' }, note: { type: 'string' } },
|
|
30
|
+
required: ['evidence', 'met', 'note'],
|
|
31
|
+
},
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
export async function judgeConversation(llm, expected, when, turns) {
|
|
35
|
+
const request = buildConversationRequest(expected, when, turns);
|
|
36
|
+
const calls = [];
|
|
37
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
38
|
+
const response = await llm.complete(request);
|
|
39
|
+
calls.push({ request, response });
|
|
40
|
+
const reply = response.structured;
|
|
41
|
+
if (reply && typeof reply.met === 'boolean')
|
|
42
|
+
return { met: reply.met, evidence: reply.evidence ?? '', note: reply.note ?? '', calls };
|
|
43
|
+
}
|
|
44
|
+
throw new Error('The conversation judge gave no verdict');
|
|
45
|
+
}
|