sdd-engine 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +178 -2
- package/dist/engine/cli.js +95 -0
- package/dist/engine/simulate.js +214 -0
- package/dist/judge/conversation.js +45 -0
- package/dist/judge/judge.js +122 -0
- package/dist/judge/outcome.js +42 -0
- package/dist/judge/regrade.js +74 -0
- package/dist/pipeline/agent.js +78 -0
- package/dist/pipeline/agentstage.js +88 -0
- package/dist/pipeline/approve.js +68 -0
- package/dist/pipeline/cli.js +261 -0
- package/dist/pipeline/derive.js +287 -0
- package/dist/pipeline/designloop.js +385 -0
- package/dist/pipeline/facts.js +108 -0
- package/dist/pipeline/mock.js +253 -0
- package/dist/pipeline/project.js +79 -0
- package/dist/pipeline/render.js +95 -0
- package/dist/pipeline/spec.js +272 -0
- package/dist/pipeline/try.js +162 -0
- package/dist/runner/batch-lib.js +157 -0
- package/dist/runner/batch.js +56 -0
- package/dist/runner/cli.js +61 -0
- package/dist/runner/llm/claude-code.js +123 -0
- package/dist/runner/llm/index.js +12 -0
- package/dist/runner/llm/types.js +17 -0
- package/dist/runner/reader/answer.js +76 -0
- package/dist/runner/reader/extract.js +106 -0
- package/dist/runner/reader/run.js +111 -0
- package/dist/runner/reader/skim.js +115 -0
- package/dist/runner/rng.js +22 -0
- package/dist/runner/sandbox/cli.js +60 -0
- package/dist/runner/sandbox/sandbox.js +144 -0
- package/dist/runner/sandbox/server.js +64 -0
- package/dist/runner/spec.js +84 -0
- package/dist/runner/talker/decide.js +110 -0
- package/dist/runner/talker/run.js +159 -0
- package/dist/runner/talker/world.js +36 -0
- package/dist/runner/trace.js +130 -0
- package/dist/runner/visitor/decide.js +159 -0
- package/dist/runner/visitor/observe.js +102 -0
- package/dist/runner/visitor/run.js +296 -0
- package/dist/scorer/compare.js +119 -0
- package/dist/scorer/score.js +201 -0
- package/dist/scorer/spot-check-compare.js +41 -0
- package/dist/scorer/spot-check-view.js +45 -0
- package/dist/scorer/stats.js +45 -0
- package/dist/scorer/summary.js +62 -0
- package/package.json +49 -4
|
@@ -0,0 +1,385 @@
|
|
|
1
|
+
// Step 4: design on cheap mocks, score them, revise, and keep what improved.
|
|
2
|
+
//
|
|
3
|
+
// One iteration: the designer writes a static page, the guardrails check it,
|
|
4
|
+
// the engine sends the people through it (reading, then deciding), and a
|
|
5
|
+
// developer on a clean machine follows its instructions. A revision is kept
|
|
6
|
+
// only when the people the designer may see did better. The held-back people
|
|
7
|
+
// only ever decide whether a pass is real.
|
|
8
|
+
import { cp, mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
import { simulate } from "../engine/simulate.js";
|
|
11
|
+
import { startStaticServer } from "../runner/sandbox/server.js";
|
|
12
|
+
import { verdict as intervalVerdict, wilson } from "../scorer/stats.js";
|
|
13
|
+
import { requireApproval } from "./approve.js";
|
|
14
|
+
import { buildBrief, checkAgainstRepo, designMock, mechanicalProblems, ownPlaces, reviewClaims, visibleText } from "./mock.js";
|
|
15
|
+
import { freshScratch, logCost, readCosts } from "./project.js";
|
|
16
|
+
import { agentStage } from "./agentstage.js";
|
|
17
|
+
import { startContainer, tryStage } from "./try.js";
|
|
18
|
+
// ---- Turning a stage into what the engine runs ----
|
|
19
|
+
/** Time spent waiting for the page is time not spent on it. */
|
|
20
|
+
const after = (seconds, loadSeconds, floor) => Math.max(floor, Math.round(seconds - loadSeconds));
|
|
21
|
+
export function engineReadSpec(stage, runs, loadSeconds = 0) {
|
|
22
|
+
return {
|
|
23
|
+
experiment: stage.id,
|
|
24
|
+
kind: 'comprehension',
|
|
25
|
+
goal: stage.goal,
|
|
26
|
+
entry: { from: stage.entryFrom, viewports: stage.viewports },
|
|
27
|
+
reader: { time_budget_seconds: after(stage.timeBudgetSeconds, loadSeconds, 5), answers_from: 'what_was_seen', may_scroll_back: false, screenshots: true },
|
|
28
|
+
personas: { runs_per_persona: runs },
|
|
29
|
+
questions: stage.questions.map((q) => ({ id: q.id, ask: q.ask, rubric: { required: q.required } })),
|
|
30
|
+
pass: Object.fromEntries(stage.questions.map((q) => [`${q.id}_rate`, q.threshold])),
|
|
31
|
+
budget: { iterations: 0 },
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
export function engineDecideSpec(stage, runs, loadSeconds = 0) {
|
|
35
|
+
const success = {};
|
|
36
|
+
if (stage.success.leavesTo.length > 0)
|
|
37
|
+
success.leaves_to = stage.success.leavesTo;
|
|
38
|
+
if (stage.success.clicks.length > 0)
|
|
39
|
+
success.clicks = stage.success.clicks;
|
|
40
|
+
return {
|
|
41
|
+
experiment: stage.id,
|
|
42
|
+
kind: 'task',
|
|
43
|
+
goal: stage.goal,
|
|
44
|
+
entry: { from: stage.entryFrom, viewports: stage.viewports },
|
|
45
|
+
// No instruction: nobody asked these visitors to do anything.
|
|
46
|
+
task: { goal: stage.goal, success },
|
|
47
|
+
visitor: { patience_seconds: after(stage.patienceSeconds, loadSeconds, 5), max_steps: 15, screenshots: true },
|
|
48
|
+
personas: { runs_per_persona: runs },
|
|
49
|
+
pass: { task_rate: stage.threshold },
|
|
50
|
+
budget: { iterations: 0 },
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
const pct = (x) => (Number.isNaN(x) ? 'n/a' : `${Math.round(x * 100)}%`);
|
|
54
|
+
/** How far the people the designer may see fall short. Lower is better. Held-back people are not in it. */
|
|
55
|
+
export function shortfall(score) {
|
|
56
|
+
let rate = 0;
|
|
57
|
+
let lower = 0;
|
|
58
|
+
for (const q of score.questions) {
|
|
59
|
+
rate += Math.max(0, q.threshold - (Number.isNaN(q.working.rate) ? 0 : q.working.rate));
|
|
60
|
+
lower += Math.max(0, q.threshold - q.working.lower);
|
|
61
|
+
}
|
|
62
|
+
if (score.tried) {
|
|
63
|
+
rate += Math.max(0, score.tried.threshold - (Number.isNaN(score.tried.interval.rate) ? 0 : score.tried.interval.rate));
|
|
64
|
+
lower += Math.max(0, score.tried.threshold - score.tried.interval.lower);
|
|
65
|
+
}
|
|
66
|
+
return { rate, lower };
|
|
67
|
+
}
|
|
68
|
+
/** Better only on the pass rates. The interval tie-break is left out on purpose: at screening sizes it is noise. */
|
|
69
|
+
export function improved(candidate, best) {
|
|
70
|
+
if (candidate.notRun.length > best.notRun.length)
|
|
71
|
+
return false;
|
|
72
|
+
return shortfall(candidate).rate < shortfall(best).rate - 0.02;
|
|
73
|
+
}
|
|
74
|
+
/** A pass needs every low end to clear its threshold, for the seen and the held-back people, and for the try stage. */
|
|
75
|
+
export function mockVerdict(score) {
|
|
76
|
+
if (score.notRun.length > 0)
|
|
77
|
+
return 'incomplete';
|
|
78
|
+
const all = [
|
|
79
|
+
...score.questions.flatMap((q) => [intervalVerdict(q.working, q.threshold), intervalVerdict(q.holdout, q.threshold)]),
|
|
80
|
+
...(score.tried ? [intervalVerdict(score.tried.interval, score.tried.threshold)] : []),
|
|
81
|
+
];
|
|
82
|
+
if (all.every((v) => v === 'met'))
|
|
83
|
+
return 'pass';
|
|
84
|
+
return all.includes('not met') ? 'not met' : 'inside the noise';
|
|
85
|
+
}
|
|
86
|
+
export const workingMet = (score) => score.questions.every((q) => intervalVerdict(q.working, q.threshold) === 'met') &&
|
|
87
|
+
(!score.tried || intervalVerdict(score.tried.interval, score.tried.threshold) === 'met');
|
|
88
|
+
async function leaveReasons(workdir, experiment, run) {
|
|
89
|
+
const dir = path.join(workdir, 'traces', experiment);
|
|
90
|
+
const reasons = [];
|
|
91
|
+
for (const file of await readdir(dir).catch(() => [])) {
|
|
92
|
+
const t = JSON.parse(await readFile(path.join(dir, file), 'utf8'));
|
|
93
|
+
// Working people only. What the held-back people said never reaches the designer.
|
|
94
|
+
if (t.batch !== run || t.personaSet !== 'working' || t.outcome === 'completed')
|
|
95
|
+
continue;
|
|
96
|
+
const last = t.visitorSteps?.at(-1);
|
|
97
|
+
if (last?.reason)
|
|
98
|
+
reasons.push(last.reason);
|
|
99
|
+
}
|
|
100
|
+
return reasons;
|
|
101
|
+
}
|
|
102
|
+
export async function scoreMock(options) {
|
|
103
|
+
const { layout: l, spec, llm } = options;
|
|
104
|
+
const log = options.log ?? (() => { });
|
|
105
|
+
const only = options.only ?? ['read', 'decide', 'try', 'agent'];
|
|
106
|
+
const score = { questions: [], findings: '', costUsd: 0, notRun: [] };
|
|
107
|
+
const findings = [];
|
|
108
|
+
const run = 'score';
|
|
109
|
+
await mkdir(options.evidenceDir, { recursive: true });
|
|
110
|
+
const server = options.running ? { origin: options.running.origin, close: async () => { } } : await startStaticServer(options.mockDir);
|
|
111
|
+
const entry = options.running?.entry ?? '/';
|
|
112
|
+
// What the terminal and agent stages read: the page's text, from the file or from the running build.
|
|
113
|
+
const pageHtml = async () => {
|
|
114
|
+
if (!options.running)
|
|
115
|
+
return readFile(path.join(options.mockDir, 'index.html'), 'utf8');
|
|
116
|
+
const response = await fetch(options.running.origin + entry);
|
|
117
|
+
if (!response.ok)
|
|
118
|
+
throw new Error(`The running build answered ${response.status} for ${entry}`);
|
|
119
|
+
return response.text();
|
|
120
|
+
};
|
|
121
|
+
try {
|
|
122
|
+
for (const stage of spec.stages) {
|
|
123
|
+
if (stage.kind === 'try' || stage.kind === 'agent' || !only.includes(stage.kind))
|
|
124
|
+
continue;
|
|
125
|
+
const workdir = path.join(options.evidenceDir, stage.id);
|
|
126
|
+
await rm(workdir, { recursive: true, force: true });
|
|
127
|
+
await mkdir(workdir, { recursive: true });
|
|
128
|
+
await cp(path.join(l.referee, 'personas'), path.join(workdir, 'personas'), { recursive: true });
|
|
129
|
+
const engineSpec = stage.kind === 'read' ? engineReadSpec(stage, options.runs, options.loadSeconds) : engineDecideSpec(stage, options.runs, options.loadSeconds);
|
|
130
|
+
await writeFile(path.join(workdir, 'spec.yaml'), JSON.stringify(engineSpec, null, 2) + '\n');
|
|
131
|
+
log(` ${stage.id}: ${spec.personas.length} people x ${stage.viewports.length} screens x ${options.runs} visits`);
|
|
132
|
+
// The owner's rule: local copies only, with outside requests blocked. A blocked request is recorded, not a failure.
|
|
133
|
+
await simulate(['--workdir', workdir, '--target', server.origin, '--entry', entry, '--run', run, '--runs', String(options.runs), '--budget-usd', Math.max(0.05, options.budgetUsd - score.costUsd).toFixed(2), '--outside-requests', 'block'], { llm, log: () => { }, logError: () => { } });
|
|
134
|
+
const report = JSON.parse(await readFile(path.join(workdir, 'reports', `${run}.json`), 'utf8'));
|
|
135
|
+
score.costUsd += report.costUsd;
|
|
136
|
+
if (report.verdict === 'incomplete') {
|
|
137
|
+
score.notRun.push({ stage: stage.id, why: report.error ?? 'some visits failed or the budget ran out' });
|
|
138
|
+
if (report.working.questions.length === 0)
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
const asks = stage.kind === 'read' ? Object.fromEntries(stage.questions.map((q) => [q.id, q.ask])) : { task: stage.goal };
|
|
142
|
+
for (const q of report.working.questions) {
|
|
143
|
+
const held = report.holdout.questions.find((h) => h.id === q.id);
|
|
144
|
+
score.questions.push({
|
|
145
|
+
stage: stage.id, id: q.id, ask: asks[q.id] ?? q.id, threshold: q.threshold, working: q.interval,
|
|
146
|
+
holdout: held?.interval ?? wilson(0, 0),
|
|
147
|
+
byViewport: Object.fromEntries(Object.entries(report.working.byViewport ?? {}).map(([vp, rates]) => [vp, rates[q.id] ?? NaN])),
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
const w = report.working;
|
|
151
|
+
findings.push(`## ${stage.kind === 'read' ? 'Reading' : 'Deciding'} (${stage.id})`, '');
|
|
152
|
+
findings.push(...w.questions.map((q) => `- ${asks[q.id] ?? q.id}: ${pct(q.interval.rate)} (${q.interval.passed}/${q.interval.n}), needs ${pct(q.threshold)}`));
|
|
153
|
+
if (w.missedElements?.length)
|
|
154
|
+
findings.push('', 'Missed most:', ...w.missedElements.filter((m) => m.missedRate > 0).map((m) => `- "${m.element}" missed by ${pct(m.missedRate)}`));
|
|
155
|
+
if (w.stoppedIn && Object.keys(w.stoppedIn).length > 0)
|
|
156
|
+
findings.push('', 'Where they were when they stopped:', ...Object.entries(w.stoppedIn).map(([name, x]) => `- ${name}: ${pct(x)}`));
|
|
157
|
+
if (w.failingAnswers?.length)
|
|
158
|
+
findings.push('', 'A sample of failing answers:', ...w.failingAnswers.slice(0, 8).map((f) => `- [${asks[f.question] ?? f.question}] ${f.answer.slice(0, 300)}\n - Judge: ${f.judgeNote}`));
|
|
159
|
+
if (stage.kind === 'decide') {
|
|
160
|
+
const reasons = await leaveReasons(workdir, stage.id, run);
|
|
161
|
+
if (reasons.length > 0)
|
|
162
|
+
findings.push('', 'Why some of them left without acting:', ...reasons.slice(0, 10).map((r) => `- ${r.slice(0, 300)}`));
|
|
163
|
+
}
|
|
164
|
+
findings.push('');
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
finally {
|
|
168
|
+
await server.close();
|
|
169
|
+
}
|
|
170
|
+
const tryDef = spec.stages.find((s) => s.kind === 'try');
|
|
171
|
+
if (tryDef && only.includes('try') && options.tryCount > 0) {
|
|
172
|
+
log(` ${tryDef.id}: ${options.tryCount} developers on clean machines`);
|
|
173
|
+
const html = await pageHtml();
|
|
174
|
+
const stamp = Date.now().toString(36);
|
|
175
|
+
const start = options.startMachine ?? ((stage, n) => startContainer(stage, `sdd-try-${stamp}-${n}`));
|
|
176
|
+
try {
|
|
177
|
+
const result = await tryStage(tryDef, visibleText(html), llm, options.tryCount, (n) => start(tryDef, n));
|
|
178
|
+
score.costUsd += result.costUsd;
|
|
179
|
+
score.tried = { stage: tryDef.id, threshold: tryDef.threshold, interval: wilson(result.passed, result.n), result };
|
|
180
|
+
await writeFile(path.join(options.evidenceDir, `${tryDef.id}.json`), JSON.stringify(result, null, 2) + '\n');
|
|
181
|
+
findings.push(`## Trying, at a terminal (${tryDef.id})`, '', `- ${result.passed} of ${result.n} got there, needs ${pct(tryDef.threshold)}`);
|
|
182
|
+
const failed = result.runs.filter((r) => !r.met);
|
|
183
|
+
if (failed.length > 0) {
|
|
184
|
+
findings.push('', 'What the ones who did not get there ran:');
|
|
185
|
+
for (const r of failed.slice(0, 4)) {
|
|
186
|
+
findings.push(`- ${r.steps.map((s) => `\`${s.command.slice(0, 160)}\` (exit ${s.exitCode})`).join(' then ') || 'nothing'}. ${r.note.slice(0, 240)}`);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
findings.push('');
|
|
190
|
+
}
|
|
191
|
+
catch (err) {
|
|
192
|
+
score.notRun.push({ stage: tryDef.id, why: err.message });
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
const agentDef = spec.stages.find((s) => s.kind === 'agent');
|
|
196
|
+
if (agentDef && only.includes('agent') && options.runs > 0) {
|
|
197
|
+
if (!options.ledger) {
|
|
198
|
+
score.notRun.push({ stage: agentDef.id, why: 'no ledger was given to check replies against' });
|
|
199
|
+
}
|
|
200
|
+
else {
|
|
201
|
+
log(` ${agentDef.id}: ${agentDef.requests.length} requests x ${options.runs} agents`);
|
|
202
|
+
const html = await pageHtml();
|
|
203
|
+
const result = await agentStage(agentDef, visibleText(html), options.ledger, llm, options.runs);
|
|
204
|
+
score.costUsd += result.costUsd;
|
|
205
|
+
await writeFile(path.join(options.evidenceDir, `${agentDef.id}.json`), JSON.stringify(result, null, 2) + '\n');
|
|
206
|
+
score.questions.push({ stage: agentDef.id, id: 'agent', ask: agentDef.goal, threshold: agentDef.threshold, working: result.working, holdout: result.holdout });
|
|
207
|
+
const errored = result.runs.filter((r) => r.note.startsWith('error:')).length;
|
|
208
|
+
if (errored > 0)
|
|
209
|
+
score.notRun.push({ stage: agentDef.id, why: `${errored} agent replies failed to run` });
|
|
210
|
+
// Only the requests the designer may see. What held-back requests produced stays with the referee.
|
|
211
|
+
const shown = result.runs.filter((r) => !r.holdout);
|
|
212
|
+
findings.push(`## AI agents asked by their developer (${agentDef.id})`, '', `- ${result.working.passed} of ${result.working.n} replies counted, needs ${pct(agentDef.threshold)}`);
|
|
213
|
+
const failed = shown.filter((r) => !r.met);
|
|
214
|
+
if (failed.length > 0) {
|
|
215
|
+
findings.push('', 'Replies that did not count:');
|
|
216
|
+
for (const r of failed.slice(0, 8)) {
|
|
217
|
+
const ask = agentDef.requests.find((q) => q.id === r.request)?.ask ?? r.request;
|
|
218
|
+
const why = !r.fits ? 'the product does not fit this and the agent recommended it anyway'
|
|
219
|
+
: [!r.recommended ? 'the agent did not recommend it' : '', r.missing.length ? `left out: ${r.missing.join('; ')}` : '', r.falseStatements.length ? `said something untrue: ${r.falseStatements.join('; ')}` : ''].filter(Boolean).join('. ');
|
|
220
|
+
findings.push(`- Asked "${ask.slice(0, 160)}": ${why}. The agent said: ${r.reply.slice(0, 300)}`);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
findings.push('');
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
score.findings = findings.join('\n');
|
|
227
|
+
score.costUsd = Math.round(score.costUsd * 10000) / 10000;
|
|
228
|
+
return score;
|
|
229
|
+
}
|
|
230
|
+
const GUARDRAIL_TRIES = 3;
|
|
231
|
+
/**
|
|
232
|
+
* What the designer hears about the people it is not shown: which questions they
|
|
233
|
+
* fall short on, and nothing else. No rates, no words of theirs, nothing about who they are.
|
|
234
|
+
* Without this a page can keep improving for the seen people while it loses everyone else.
|
|
235
|
+
*/
|
|
236
|
+
export function unseenNote(score) {
|
|
237
|
+
const short = score.questions.filter((q) => q.holdout.n > 0 && q.holdout.rate < q.threshold);
|
|
238
|
+
if (short.length === 0)
|
|
239
|
+
return '';
|
|
240
|
+
return [
|
|
241
|
+
'',
|
|
242
|
+
'## Visitors you are not shown',
|
|
243
|
+
'',
|
|
244
|
+
'Other kinds of visitor in this audience also came. They fell short of what is needed on:',
|
|
245
|
+
...short.map((q) => `- ${q.ask}`),
|
|
246
|
+
'',
|
|
247
|
+
'You are not told who they are. Do not guess at them and do not aim anything at one kind of visitor. Make the page work for someone in this audience who arrives from somewhere else, knowing less or wanting something slightly different from the people you were shown.',
|
|
248
|
+
'',
|
|
249
|
+
].join('\n');
|
|
250
|
+
}
|
|
251
|
+
/** A design scored earlier, read back so the loop can carry on from it. */
|
|
252
|
+
export async function loadScored(l, iteration) {
|
|
253
|
+
const dir = path.join(l.design, `iter-${iteration}`);
|
|
254
|
+
const saved = JSON.parse(await readFile(path.join(dir, 'score.json'), 'utf8'));
|
|
255
|
+
await readFile(path.join(dir, 'index.html'), 'utf8');
|
|
256
|
+
return { iteration, score: saved, verdict: saved.verdict, dir };
|
|
257
|
+
}
|
|
258
|
+
/** What step 4 has spent so far, from the log. The log is the only source. */
|
|
259
|
+
export async function designSpend(l) {
|
|
260
|
+
return (await readCosts(l)).filter((c) => c.step === 'design').reduce((s, c) => s + c.usd, 0);
|
|
261
|
+
}
|
|
262
|
+
export async function runDesignLoop(options) {
|
|
263
|
+
const { layout: l, spec, ledger, llm } = options;
|
|
264
|
+
const log = options.log ?? (() => { });
|
|
265
|
+
const approval = await requireApproval(l);
|
|
266
|
+
const maxIterations = options.maxIterations ?? 6;
|
|
267
|
+
const runs = options.runs ?? 4;
|
|
268
|
+
const tryCount = options.tryCount ?? 5;
|
|
269
|
+
const places = ownPlaces(spec, ledger, options.ownPlaces ?? []);
|
|
270
|
+
const baseBrief = buildBrief(spec, ledger, places);
|
|
271
|
+
await writeFile(path.join(l.brief, 'brief.md'), baseBrief);
|
|
272
|
+
// Statements beyond the ledger that were checked against the repository during this run.
|
|
273
|
+
const verifiedFile = path.join(l.evidence, 'verified-claims.json');
|
|
274
|
+
const verified = await readFile(verifiedFile, 'utf8').then((t) => JSON.parse(t), () => []);
|
|
275
|
+
const briefNow = () => (verified.length === 0 ? baseBrief : `${baseBrief}\n# Also checked against the repository during this run\n\n${verified.map((v) => `- ${v.claim}`).join('\n')}\n`);
|
|
276
|
+
const iterations = [];
|
|
277
|
+
// Carrying on from an earlier design: it is the one to beat, and new designs are numbered after the last one made.
|
|
278
|
+
let best = options.resumeFrom === undefined ? undefined : await loadScored(l, options.resumeFrom);
|
|
279
|
+
const first = options.resumeFrom === undefined ? 0 : (options.startAt ?? options.resumeFrom + 1);
|
|
280
|
+
let stale = 0;
|
|
281
|
+
if (best && options.rescoreFirst) {
|
|
282
|
+
// The referee changed since this design was scored (new held-back people, say). Score it again so new designs are compared like for like.
|
|
283
|
+
log(`--- Design ${best.iteration}, scored again ---`);
|
|
284
|
+
const again = await scoreMock({
|
|
285
|
+
layout: l, spec, mockDir: best.dir, evidenceDir: path.join(l.evidence, 'design', `iter-${best.iteration}-rescored`),
|
|
286
|
+
runs, tryCount, budgetUsd: Math.max(0.5, approval.budgetUsd - (await designSpend(l))), llm, ledger, startMachine: options.startMachine, log,
|
|
287
|
+
});
|
|
288
|
+
await logCost(l, { step: 'design', what: `simulation, design ${best.iteration} scored again`, usd: again.costUsd, durationMs: 0, detail: { runs, tryCount } });
|
|
289
|
+
const v = mockVerdict(again);
|
|
290
|
+
const s = shortfall(again).rate;
|
|
291
|
+
await writeFile(path.join(best.dir, 'score-rescored.json'), JSON.stringify({ verdict: v, shortfall: s, ...again, tried: again.tried && { ...again.tried, result: undefined, passed: again.tried.result.passed, n: again.tried.result.n } }, null, 2) + '\n');
|
|
292
|
+
log(` verdict: ${v}, shortfall ${s.toFixed(2)}`);
|
|
293
|
+
best = { ...best, score: again, verdict: v };
|
|
294
|
+
}
|
|
295
|
+
const left = async () => approval.budgetUsd - (await designSpend(l));
|
|
296
|
+
const finish = async (stop) => ({ stop, iterations, best, spentUsd: Math.round((await designSpend(l)) * 100) / 100 });
|
|
297
|
+
for (let iteration = first; iteration < first + maxIterations; iteration++) {
|
|
298
|
+
if (best?.verdict === 'pass')
|
|
299
|
+
return finish('pass');
|
|
300
|
+
if ((await left()) <= 0.5)
|
|
301
|
+
return finish('budget');
|
|
302
|
+
if (stale >= 2)
|
|
303
|
+
return finish('plateau');
|
|
304
|
+
log(`--- Design ${iteration} ---`);
|
|
305
|
+
const dir = path.join(l.design, `iter-${iteration}`);
|
|
306
|
+
await rm(dir, { recursive: true, force: true });
|
|
307
|
+
await mkdir(dir, { recursive: true });
|
|
308
|
+
let cost = 0;
|
|
309
|
+
// Design, then guardrails. A page that fails them is sent back, not scored.
|
|
310
|
+
let html;
|
|
311
|
+
let notes = '';
|
|
312
|
+
let rejected;
|
|
313
|
+
for (let attempt = 1; attempt <= GUARDRAIL_TRIES && !html; attempt++) {
|
|
314
|
+
const previous = best ? { html: await readFile(path.join(best.dir, 'index.html'), 'utf8'), findings: best.score.findings + unseenNote(best.score) } : undefined;
|
|
315
|
+
const design = await designMock(llm, { brief: briefNow(), iteration, previous, rejected });
|
|
316
|
+
cost += design.costUsd;
|
|
317
|
+
await logCost(l, { step: 'design', what: `designer, design ${iteration} attempt ${attempt}`, usd: design.costUsd, durationMs: 0 });
|
|
318
|
+
const problems = mechanicalProblems(design.html, places);
|
|
319
|
+
if (problems.length === 0) {
|
|
320
|
+
const review = await reviewClaims(llm, design.html, ledger, verified.map((v) => v.claim));
|
|
321
|
+
cost += review.costUsd;
|
|
322
|
+
await logCost(l, { step: 'design', what: `claims review, design ${iteration} attempt ${attempt}`, usd: review.costUsd, durationMs: 0 });
|
|
323
|
+
let violations = review.violations;
|
|
324
|
+
// "True" means the repository supports it, not that it is on a list. Look up what the ledger does not cover.
|
|
325
|
+
if (violations.length > 0 && options.agent && (options.sourceDirs?.length ?? 0) > 0) {
|
|
326
|
+
const check = await checkAgainstRepo(options.agent, violations, ledger, options.sourceDirs, await freshScratch(l, 'claims-check'), Math.min(1.5, Math.max(0.2, await left())));
|
|
327
|
+
cost += check.costUsd;
|
|
328
|
+
await logCost(l, { step: 'design', what: `repository check of ${violations.length} statement(s), design ${iteration} attempt ${attempt}`, usd: check.costUsd, durationMs: 0, detail: { supported: check.supported.length } });
|
|
329
|
+
if (check.supported.length > 0) {
|
|
330
|
+
verified.push(...check.supported.map(({ claim, file, line, quote }) => ({ claim, file, line, quote })));
|
|
331
|
+
await writeFile(verifiedFile, JSON.stringify(verified, null, 2) + '\n');
|
|
332
|
+
log(` ${check.supported.length} statement(s) beyond the ledger checked out against the repository`);
|
|
333
|
+
}
|
|
334
|
+
violations = check.unsupported;
|
|
335
|
+
}
|
|
336
|
+
problems.push(...violations.map((v) => `The page says "${v.statement}". ${v.why}`));
|
|
337
|
+
}
|
|
338
|
+
if (problems.length === 0) {
|
|
339
|
+
html = design.html;
|
|
340
|
+
notes = design.notes;
|
|
341
|
+
}
|
|
342
|
+
else {
|
|
343
|
+
rejected = problems;
|
|
344
|
+
await writeFile(path.join(dir, `rejected-${attempt}.html`), design.html);
|
|
345
|
+
await writeFile(path.join(dir, `rejected-${attempt}.txt`), problems.join('\n') + '\n');
|
|
346
|
+
log(` thrown out by guardrails (${problems.length}): ${problems[0].slice(0, 140)}`);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
if (!html) {
|
|
350
|
+
iterations.push({ iteration, outcome: 'thrown out by guardrails', rejected, costUsd: cost });
|
|
351
|
+
stale++;
|
|
352
|
+
continue;
|
|
353
|
+
}
|
|
354
|
+
await writeFile(path.join(dir, 'index.html'), html);
|
|
355
|
+
await writeFile(path.join(dir, 'notes.md'), notes + '\n');
|
|
356
|
+
const score = await scoreMock({
|
|
357
|
+
layout: l, spec, mockDir: dir, evidenceDir: path.join(l.evidence, 'design', `iter-${iteration}`),
|
|
358
|
+
runs, tryCount, budgetUsd: Math.max(0.5, await left()), llm, ledger, startMachine: options.startMachine, log,
|
|
359
|
+
});
|
|
360
|
+
cost += score.costUsd;
|
|
361
|
+
await logCost(l, { step: 'design', what: `simulation, design ${iteration}`, usd: score.costUsd, durationMs: 0, detail: { runs, tryCount } });
|
|
362
|
+
const v = mockVerdict(score);
|
|
363
|
+
const s = shortfall(score).rate;
|
|
364
|
+
await writeFile(path.join(dir, 'score.json'), JSON.stringify({ verdict: v, shortfall: s, ...score, tried: score.tried && { ...score.tried, result: undefined, passed: score.tried.result.passed, n: score.tried.result.n } }, null, 2) + '\n');
|
|
365
|
+
log(` verdict: ${v}, shortfall ${s.toFixed(2)}, $${cost.toFixed(2)}`);
|
|
366
|
+
if (!best) {
|
|
367
|
+
best = { iteration, score, verdict: v, dir };
|
|
368
|
+
iterations.push({ iteration, outcome: 'first', verdict: v, shortfall: s, notes, costUsd: cost });
|
|
369
|
+
}
|
|
370
|
+
else if (improved(score, best.score)) {
|
|
371
|
+
best = { iteration, score, verdict: v, dir };
|
|
372
|
+
stale = 0;
|
|
373
|
+
iterations.push({ iteration, outcome: 'kept', verdict: v, shortfall: s, notes, costUsd: cost });
|
|
374
|
+
}
|
|
375
|
+
else {
|
|
376
|
+
stale++;
|
|
377
|
+
iterations.push({ iteration, outcome: 'reverted: no improvement', verdict: v, shortfall: s, notes, costUsd: cost });
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
if (!best)
|
|
381
|
+
return finish('no usable design');
|
|
382
|
+
if (best.verdict === 'pass')
|
|
383
|
+
return finish('pass');
|
|
384
|
+
return finish(stale >= 2 ? 'plateau' : 'iterations');
|
|
385
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
// The claims ledger: every statement a designer or builder may make about the
|
|
2
|
+
// product, each tied to where it comes from.
|
|
3
|
+
//
|
|
4
|
+
// This is how "true claims only" is enforced. The deriver reads the sources
|
|
5
|
+
// and writes the ledger. Code then checks every citation: the file exists and
|
|
6
|
+
// the quoted words are really there, near the line given. Later, anything a
|
|
7
|
+
// mock says about the product has to be in the ledger.
|
|
8
|
+
import { readFile } from 'node:fs/promises';
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
import { Issues, isObject, list, text } from "./spec.js";
|
|
11
|
+
const SLUG = /^[a-z][a-z0-9-]{1,59}$/;
|
|
12
|
+
export function parseLedger(input) {
|
|
13
|
+
const issues = new Issues();
|
|
14
|
+
if (!isObject(input))
|
|
15
|
+
throw new Error('The ledger must be a JSON object.');
|
|
16
|
+
const facts = list(issues, 'facts', input.facts, 5, 120, (item, where) => {
|
|
17
|
+
const f = isObject(item) ? item : {};
|
|
18
|
+
if (!isObject(item))
|
|
19
|
+
issues.add(where, 'must be an object');
|
|
20
|
+
const id = typeof f.id === 'string' && SLUG.test(f.id) ? f.id : '';
|
|
21
|
+
if (!id)
|
|
22
|
+
issues.add(`${where}.id`, 'must be lowercase letters, digits and dashes, starting with a letter');
|
|
23
|
+
const claim = text(issues, `${where}.claim`, f.claim, 8, 400);
|
|
24
|
+
const s = isObject(f.source) ? f.source : {};
|
|
25
|
+
let source;
|
|
26
|
+
if (s.kind === 'file') {
|
|
27
|
+
if (typeof s.line !== 'number' || !Number.isInteger(s.line) || s.line < 1)
|
|
28
|
+
issues.add(`${where}.source.line`, 'must be a line number');
|
|
29
|
+
source = {
|
|
30
|
+
kind: 'file',
|
|
31
|
+
file: text(issues, `${where}.source.file`, s.file, 1, 400),
|
|
32
|
+
line: typeof s.line === 'number' ? s.line : 1,
|
|
33
|
+
quote: text(issues, `${where}.source.quote`, s.quote, 4, 400),
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
else if (s.kind === 'measured') {
|
|
37
|
+
source = {
|
|
38
|
+
kind: 'measured',
|
|
39
|
+
how: text(issues, `${where}.source.how`, s.how, 8, 400),
|
|
40
|
+
reading: text(issues, `${where}.source.reading`, s.reading, 1, 400),
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
issues.add(`${where}.source.kind`, 'must be "file" or "measured"');
|
|
45
|
+
source = { kind: 'measured', how: '', reading: '' };
|
|
46
|
+
}
|
|
47
|
+
return { id, claim, source };
|
|
48
|
+
});
|
|
49
|
+
const notTrue = list(issues, 'notTrue', input.notTrue ?? [], 0, 40, (item, where) => {
|
|
50
|
+
const n = isObject(item) ? item : {};
|
|
51
|
+
if (!isObject(item))
|
|
52
|
+
issues.add(where, 'must be an object');
|
|
53
|
+
return { claim: text(issues, `${where}.claim`, n.claim, 8, 400), why: text(issues, `${where}.why`, n.why, 8, 500) };
|
|
54
|
+
});
|
|
55
|
+
const ids = facts.map((f) => f.id);
|
|
56
|
+
if (new Set(ids).size !== ids.length)
|
|
57
|
+
issues.add('facts', 'ids must be unique');
|
|
58
|
+
issues.throwIfAny('The ledger');
|
|
59
|
+
return { schema: 'sdd-facts/1', facts, notTrue };
|
|
60
|
+
}
|
|
61
|
+
const squash = (s) => s.replace(/\s+/g, ' ').trim().toLowerCase();
|
|
62
|
+
/** How far from the cited line the quote may sit. Line numbers drift by a little; this is not a search. */
|
|
63
|
+
const LINE_SLACK = 6;
|
|
64
|
+
/**
|
|
65
|
+
* Checks every file citation against the real files. Returns the problems;
|
|
66
|
+
* an empty list means every quote was found where the ledger says it is.
|
|
67
|
+
* `file` may be absolute, or relative to any of the source folders.
|
|
68
|
+
*/
|
|
69
|
+
export async function checkCitations(ledger, sourceDirs) {
|
|
70
|
+
const problems = [];
|
|
71
|
+
const cache = new Map();
|
|
72
|
+
const load = async (file) => {
|
|
73
|
+
const candidates = path.isAbsolute(file) ? [file] : sourceDirs.map((dir) => path.resolve(dir, file));
|
|
74
|
+
for (const candidate of candidates) {
|
|
75
|
+
// A citation may only point inside a source folder.
|
|
76
|
+
const inside = sourceDirs.some((dir) => {
|
|
77
|
+
const rel = path.relative(path.resolve(dir), candidate);
|
|
78
|
+
return !rel.startsWith('..') && !path.isAbsolute(rel);
|
|
79
|
+
});
|
|
80
|
+
if (!inside)
|
|
81
|
+
continue;
|
|
82
|
+
if (!cache.has(candidate)) {
|
|
83
|
+
cache.set(candidate, await readFile(candidate, 'utf8').then((t) => t.split('\n'), () => null));
|
|
84
|
+
}
|
|
85
|
+
const lines = cache.get(candidate);
|
|
86
|
+
if (lines)
|
|
87
|
+
return lines;
|
|
88
|
+
}
|
|
89
|
+
return null;
|
|
90
|
+
};
|
|
91
|
+
for (const fact of ledger.facts) {
|
|
92
|
+
if (fact.source.kind !== 'file')
|
|
93
|
+
continue;
|
|
94
|
+
const { file, line, quote } = fact.source;
|
|
95
|
+
const lines = await load(file);
|
|
96
|
+
if (!lines) {
|
|
97
|
+
problems.push(`facts "${fact.id}": ${file} is not a file in the sources`);
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
const from = Math.max(0, line - 1 - LINE_SLACK);
|
|
101
|
+
const to = Math.min(lines.length, line + LINE_SLACK);
|
|
102
|
+
const near = squash(lines.slice(from, to).join(' '));
|
|
103
|
+
if (!near.includes(squash(quote))) {
|
|
104
|
+
problems.push(`facts "${fact.id}": the quote is not at ${file}:${line} (looked ${LINE_SLACK} lines either side)`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return problems;
|
|
108
|
+
}
|