sdd-engine 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +165 -2
- package/dist/engine/cli.js +95 -0
- package/dist/engine/simulate.js +214 -0
- package/dist/judge/conversation.js +45 -0
- package/dist/judge/judge.js +122 -0
- package/dist/judge/outcome.js +42 -0
- package/dist/judge/regrade.js +74 -0
- package/dist/pipeline/agent.js +78 -0
- package/dist/pipeline/agentstage.js +88 -0
- package/dist/pipeline/approve.js +68 -0
- package/dist/pipeline/cli.js +261 -0
- package/dist/pipeline/derive.js +287 -0
- package/dist/pipeline/designloop.js +385 -0
- package/dist/pipeline/facts.js +108 -0
- package/dist/pipeline/mock.js +253 -0
- package/dist/pipeline/project.js +79 -0
- package/dist/pipeline/render.js +95 -0
- package/dist/pipeline/spec.js +272 -0
- package/dist/pipeline/try.js +162 -0
- package/dist/runner/batch-lib.js +157 -0
- package/dist/runner/batch.js +56 -0
- package/dist/runner/cli.js +61 -0
- package/dist/runner/llm/claude-code.js +123 -0
- package/dist/runner/llm/index.js +12 -0
- package/dist/runner/llm/types.js +17 -0
- package/dist/runner/reader/answer.js +76 -0
- package/dist/runner/reader/extract.js +106 -0
- package/dist/runner/reader/run.js +111 -0
- package/dist/runner/reader/skim.js +115 -0
- package/dist/runner/rng.js +22 -0
- package/dist/runner/sandbox/cli.js +60 -0
- package/dist/runner/sandbox/sandbox.js +144 -0
- package/dist/runner/sandbox/server.js +64 -0
- package/dist/runner/spec.js +84 -0
- package/dist/runner/talker/decide.js +110 -0
- package/dist/runner/talker/run.js +159 -0
- package/dist/runner/talker/world.js +36 -0
- package/dist/runner/trace.js +130 -0
- package/dist/runner/visitor/decide.js +159 -0
- package/dist/runner/visitor/observe.js +102 -0
- package/dist/runner/visitor/run.js +296 -0
- package/dist/scorer/compare.js +119 -0
- package/dist/scorer/score.js +201 -0
- package/dist/scorer/spot-check-compare.js +41 -0
- package/dist/scorer/spot-check-view.js +45 -0
- package/dist/scorer/stats.js +45 -0
- package/dist/scorer/summary.js +62 -0
- package/package.json +49 -4
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// Judges whether a visitor's session ended with the expected outcome on
|
|
2
|
+
// screen. Part of the referee.
|
|
3
|
+
//
|
|
4
|
+
// Used when a task's success cannot be checked mechanically (no known
|
|
5
|
+
// endpoint, URL, or exact text), which is the usual case for an app built
|
|
6
|
+
// from a prose spec: "the new plant appears in the list with its next
|
|
7
|
+
// watering date". The judge sees the expected outcome and the final screen.
|
|
8
|
+
// It does not see the persona, the visitor's reasoning, or the steps taken.
|
|
9
|
+
const SYSTEM = `You check whether a web app's screen shows that a task was actually completed. You are strict.
|
|
10
|
+
|
|
11
|
+
You get the expected outcome and the text on the screen where a visitor's session ended. Decide whether the screen shows the expected outcome.
|
|
12
|
+
|
|
13
|
+
Rules:
|
|
14
|
+
- Evidence first. Quote the exact words on the screen that show the outcome. If there are none, write "none". Decide only after quoting.
|
|
15
|
+
- The outcome must be visible on this screen. A form that is filled in but not submitted is not the outcome. A button that would produce the outcome is not the outcome.
|
|
16
|
+
- An error message, a validation message, or a login wall means the outcome is not met.
|
|
17
|
+
- Do not give credit for what probably happened. Judge only what the screen shows.
|
|
18
|
+
- The note is one short sentence: what shows the outcome, or what is missing.`;
|
|
19
|
+
export function buildOutcomeRequest(expected, screen) {
|
|
20
|
+
return {
|
|
21
|
+
role: 'judge',
|
|
22
|
+
system: SYSTEM,
|
|
23
|
+
prompt: `EXPECTED OUTCOME\n${expected}\n\nFINAL SCREEN\nPage: "${screen.title}" at ${screen.url}\n${screen.text.length ? screen.text.map((t) => `- ${t}`).join('\n') : '(no text on screen)'}\n\nDoes the final screen show the expected outcome?`,
|
|
24
|
+
schema: {
|
|
25
|
+
type: 'object',
|
|
26
|
+
properties: { evidence: { type: 'string' }, met: { type: 'boolean' }, note: { type: 'string' } },
|
|
27
|
+
required: ['evidence', 'met', 'note'],
|
|
28
|
+
},
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
export async function judgeOutcome(llm, expected, screen) {
|
|
32
|
+
const request = buildOutcomeRequest(expected, screen);
|
|
33
|
+
const calls = [];
|
|
34
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
35
|
+
const response = await llm.complete(request);
|
|
36
|
+
calls.push({ request, response });
|
|
37
|
+
const reply = response.structured;
|
|
38
|
+
if (reply && typeof reply.met === 'boolean')
|
|
39
|
+
return { met: reply.met, evidence: reply.evidence ?? '', note: reply.note ?? '', calls };
|
|
40
|
+
}
|
|
41
|
+
throw new Error('The outcome judge gave no verdict');
|
|
42
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
// Regrades the stored answers of a batch with the current judge.
|
|
2
|
+
//
|
|
3
|
+
// node judge/regrade.ts --batch <id> --label <name> --budget-usd <n> [--concurrency 6]
|
|
4
|
+
//
|
|
5
|
+
// Traces are never edited. Each regrade is written next to them, to
|
|
6
|
+
// grades/<experiment>/<batch>/<label>/<runId>.json, with the judge's full
|
|
7
|
+
// prompts and replies. The scorer reads them with --grades <label>.
|
|
8
|
+
// Safe to rerun: runs already regraded under the label are skipped.
|
|
9
|
+
import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
|
|
10
|
+
import path from 'node:path';
|
|
11
|
+
import { parseArgs } from 'node:util';
|
|
12
|
+
import { createLLM } from "../runner/llm/index.js";
|
|
13
|
+
import { loadSpec } from "../runner/spec.js";
|
|
14
|
+
import { JUDGE_VERSION, judgeAnswers } from "./judge.js";
|
|
15
|
+
const repoRoot = path.resolve(import.meta.dirname, '..');
|
|
16
|
+
const { values } = parseArgs({
|
|
17
|
+
options: {
|
|
18
|
+
spec: { type: 'string', default: 'specs/sdd-site.spec.yaml' },
|
|
19
|
+
batch: { type: 'string' },
|
|
20
|
+
label: { type: 'string' },
|
|
21
|
+
concurrency: { type: 'string', default: '6' },
|
|
22
|
+
'budget-usd': { type: 'string' },
|
|
23
|
+
},
|
|
24
|
+
});
|
|
25
|
+
if (!values.batch || !values.label || !values['budget-usd']) {
|
|
26
|
+
console.error('Usage: node judge/regrade.ts --batch <id> --label <name> --budget-usd <n> [--concurrency n]');
|
|
27
|
+
process.exit(2);
|
|
28
|
+
}
|
|
29
|
+
const budgetUsd = Number(values['budget-usd']);
|
|
30
|
+
const spec = await loadSpec(path.resolve(repoRoot, values.spec));
|
|
31
|
+
const tracesDir = path.join(repoRoot, 'traces', spec.experiment);
|
|
32
|
+
const outDir = path.join(repoRoot, 'grades', spec.experiment, values.batch, values.label);
|
|
33
|
+
await mkdir(outDir, { recursive: true });
|
|
34
|
+
const done = new Set(await readdir(outDir));
|
|
35
|
+
const todo = [];
|
|
36
|
+
for (const file of (await readdir(tracesDir)).sort()) {
|
|
37
|
+
if (!file.endsWith('.json') || done.has(file))
|
|
38
|
+
continue;
|
|
39
|
+
const trace = JSON.parse(await readFile(path.join(tracesDir, file), 'utf8'));
|
|
40
|
+
if (trace.batch === values.batch)
|
|
41
|
+
todo.push(trace);
|
|
42
|
+
}
|
|
43
|
+
console.log(`Regrade "${values.label}" of batch "${values.batch}" with judge ${JUDGE_VERSION}: ${done.size} done, ${todo.length} to do`);
|
|
44
|
+
const llm = createLLM();
|
|
45
|
+
let next = 0, finished = 0, failed = 0, spent = 0, budgetHit = false;
|
|
46
|
+
async function worker() {
|
|
47
|
+
while (next < todo.length) {
|
|
48
|
+
if (spent >= budgetUsd) {
|
|
49
|
+
budgetHit = true;
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
const trace = todo[next++];
|
|
53
|
+
try {
|
|
54
|
+
const { grades, calls } = await judgeAnswers(llm, spec.questions, trace.answers);
|
|
55
|
+
const costUsd = calls.reduce((s, c) => s + c.response.costUsd, 0);
|
|
56
|
+
spent += costUsd;
|
|
57
|
+
await writeFile(path.join(outDir, `${trace.runId}.json`), JSON.stringify({ runId: trace.runId, batch: trace.batch, label: values.label, judgeVersion: JUDGE_VERSION, grades, llmCalls: calls, costUsd }, null, 2) + '\n', { flag: 'wx' });
|
|
58
|
+
}
|
|
59
|
+
catch (err) {
|
|
60
|
+
failed++;
|
|
61
|
+
console.error(`FAILED ${trace.runId}: ${String(err).slice(0, 500)}`);
|
|
62
|
+
}
|
|
63
|
+
if (++finished % 40 === 0 || finished === todo.length)
|
|
64
|
+
console.log(` ${finished}/${todo.length}, ${failed} failed, $${spent.toFixed(2)}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
await Promise.all(Array.from({ length: Number(values.concurrency) }, worker));
|
|
68
|
+
console.log(`Done: ${finished - failed} regraded, ${failed} failed, $${spent.toFixed(2)} spent`);
|
|
69
|
+
if (budgetHit) {
|
|
70
|
+
console.error(`STOPPED: budget of $${budgetUsd.toFixed(2)} reached.`);
|
|
71
|
+
process.exit(3);
|
|
72
|
+
}
|
|
73
|
+
if (failed > 0)
|
|
74
|
+
process.exit(1);
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// An agent session with tools, for pipeline steps that have to look and
|
|
2
|
+
// measure. The engine's own model calls have no tools at all (see
|
|
3
|
+
// runner/llm/claude-code.ts); this is the other kind.
|
|
4
|
+
//
|
|
5
|
+
// Every session has a named list of tools, a list of folders it may read
|
|
6
|
+
// beyond its own, and a dollar cap. Its cost is reported back for the log.
|
|
7
|
+
import { spawn } from 'node:child_process';
|
|
8
|
+
export const DEFAULT_AGENT_MODEL = 'claude-opus-5-5';
|
|
9
|
+
const DEFAULT_TIMEOUT_MS = 30 * 60 * 1000;
|
|
10
|
+
export function agentArgs(options, env = process.env) {
|
|
11
|
+
return [
|
|
12
|
+
'-p',
|
|
13
|
+
'--model', options.model ?? env.SDD_MODEL_DERIVER ?? DEFAULT_AGENT_MODEL,
|
|
14
|
+
'--output-format', 'json',
|
|
15
|
+
'--max-budget-usd', options.budgetUsd.toFixed(2),
|
|
16
|
+
// Only these tools exist for the session, and they run without asking.
|
|
17
|
+
'--tools', options.tools.join(','),
|
|
18
|
+
'--allowedTools', options.tools.join(','),
|
|
19
|
+
// No user settings, no MCP servers: it must not reach a memory of earlier conversations.
|
|
20
|
+
'--strict-mcp-config',
|
|
21
|
+
'--setting-sources', '',
|
|
22
|
+
'--disable-slash-commands',
|
|
23
|
+
'--no-session-persistence',
|
|
24
|
+
...options.addDirs.flatMap((dir) => ['--add-dir', dir]),
|
|
25
|
+
];
|
|
26
|
+
}
|
|
27
|
+
/** Reads the JSON the CLI prints in print mode. */
|
|
28
|
+
export function parseAgentOutput(stdout, exitCode, stderr) {
|
|
29
|
+
try {
|
|
30
|
+
const out = JSON.parse(stdout);
|
|
31
|
+
const ok = exitCode === 0 && out.is_error !== true;
|
|
32
|
+
return {
|
|
33
|
+
ok,
|
|
34
|
+
text: out.result ?? '',
|
|
35
|
+
costUsd: typeof out.total_cost_usd === 'number' ? out.total_cost_usd : 0,
|
|
36
|
+
turns: out.num_turns,
|
|
37
|
+
durationMs: out.duration_ms,
|
|
38
|
+
error: ok ? undefined : `${out.subtype ?? 'error'}: ${(out.result ?? stderr).slice(0, 600)}`,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return { ok: false, text: '', costUsd: 0, error: `claude exited ${exitCode}: ${(stderr || stdout).slice(0, 600)}` };
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
export function createClaudeAgent() {
|
|
46
|
+
return {
|
|
47
|
+
run(options) {
|
|
48
|
+
const started = Date.now();
|
|
49
|
+
return new Promise((resolve) => {
|
|
50
|
+
const child = spawn('claude', agentArgs(options), { cwd: options.cwd, stdio: ['pipe', 'pipe', 'pipe'] });
|
|
51
|
+
let out = '';
|
|
52
|
+
let err = '';
|
|
53
|
+
let timedOut = false;
|
|
54
|
+
const timer = setTimeout(() => {
|
|
55
|
+
timedOut = true;
|
|
56
|
+
child.kill('SIGTERM');
|
|
57
|
+
}, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
58
|
+
child.stdout.on('data', (d) => (out += d));
|
|
59
|
+
child.stderr.on('data', (d) => (err += d));
|
|
60
|
+
child.on('error', (e) => {
|
|
61
|
+
clearTimeout(timer);
|
|
62
|
+
resolve({ ok: false, text: '', costUsd: 0, durationMs: Date.now() - started, error: `could not start claude: ${e.message}` });
|
|
63
|
+
});
|
|
64
|
+
child.on('close', (code) => {
|
|
65
|
+
clearTimeout(timer);
|
|
66
|
+
const parsed = parseAgentOutput(out, code, err);
|
|
67
|
+
resolve({
|
|
68
|
+
...parsed,
|
|
69
|
+
durationMs: Date.now() - started,
|
|
70
|
+
ok: parsed.ok && !timedOut,
|
|
71
|
+
error: timedOut ? `${options.name} was stopped after ${Math.round((options.timeoutMs ?? DEFAULT_TIMEOUT_MS) / 60000)} minutes` : parsed.error,
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
child.stdin.end(options.prompt);
|
|
75
|
+
});
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// The "agent" stage: the visitor is an AI coding agent, not a person.
|
|
2
|
+
//
|
|
3
|
+
// A developer asks their agent for something. The agent fetches the page, reads
|
|
4
|
+
// all of its text (no layout, no images, no time limit) and replies to its
|
|
5
|
+
// developer. A judge then reads the reply: did it recommend the product for
|
|
6
|
+
// this request, did it state what a recommendation has to get right, and did
|
|
7
|
+
// it say anything the ledger does not support.
|
|
8
|
+
//
|
|
9
|
+
// Some requests are ones the product does not fit. For those the right reply
|
|
10
|
+
// is not to recommend it, so a page that oversells scores worse, not better.
|
|
11
|
+
import { wilson } from "../scorer/stats.js";
|
|
12
|
+
const AGENT_SYSTEM = `You are an AI coding agent working for a developer in their terminal. Your developer asked you for something. You searched and fetched one web page. Its text is below.
|
|
13
|
+
|
|
14
|
+
Reply to your developer the way you would in the session, in a few sentences:
|
|
15
|
+
- Say whether this is what they asked for.
|
|
16
|
+
- If it is, say whether they should try it, exactly what to run, and what they need in place first.
|
|
17
|
+
- If it is not a fit for what they asked, say so plainly and do not recommend it for that.
|
|
18
|
+
Use only what the page says about the product. Do not add anything from memory, and do not guess at what the page leaves out: if something they need to know is missing, say it is not stated.`;
|
|
19
|
+
const JUDGE_SYSTEM = `You grade one reply an AI coding agent gave its developer after reading a product's web page.
|
|
20
|
+
|
|
21
|
+
You get what the developer asked, the agent's reply, the points a recommendation has to get right, and a ledger of true statements about the product with a list of statements that are not true.
|
|
22
|
+
- recommended: true only if the reply tells the developer to try or use the product for what they asked. "It could work but I would not use it for this" is false.
|
|
23
|
+
- missing: each required point the reply does not state. Rewording is fine. A command must be given as written.
|
|
24
|
+
- falseStatements: anything the reply says about the product that the ledger does not support or that matches the "not true" list. Quote the reply. Saying that something is not stated on the page is not a false statement.
|
|
25
|
+
Grade the reply only. Do not reward or punish style.`;
|
|
26
|
+
const JUDGE_SCHEMA = {
|
|
27
|
+
type: 'object',
|
|
28
|
+
properties: {
|
|
29
|
+
recommended: { type: 'boolean' },
|
|
30
|
+
missing: { type: 'array', items: { type: 'string' } },
|
|
31
|
+
falseStatements: { type: 'array', items: { type: 'string' } },
|
|
32
|
+
note: { type: 'string', description: 'One line on why.' },
|
|
33
|
+
},
|
|
34
|
+
required: ['recommended', 'missing', 'falseStatements', 'note'],
|
|
35
|
+
additionalProperties: false,
|
|
36
|
+
};
|
|
37
|
+
/** Whether one graded reply counts. Decided by code from what the judge reported. */
|
|
38
|
+
export function agentMet(fits, graded) {
|
|
39
|
+
if (!fits)
|
|
40
|
+
return !graded.recommended;
|
|
41
|
+
return graded.recommended && graded.missing.length === 0 && graded.falseStatements.length === 0;
|
|
42
|
+
}
|
|
43
|
+
async function once(stage, request, n, pageText, ledger, llm) {
|
|
44
|
+
const base = { request: request.id, holdout: request.holdout, fits: request.fits, n };
|
|
45
|
+
try {
|
|
46
|
+
const answer = await llm.complete({
|
|
47
|
+
role: 'persona',
|
|
48
|
+
system: AGENT_SYSTEM,
|
|
49
|
+
prompt: [`# What your developer asked`, '', request.ask, '', '# The page you fetched', '', pageText].join('\n'),
|
|
50
|
+
schema: { type: 'object', properties: { reply: { type: 'string' } }, required: ['reply'], additionalProperties: false },
|
|
51
|
+
});
|
|
52
|
+
const reply = answer.structured?.reply ?? answer.text;
|
|
53
|
+
const graded = await llm.complete({
|
|
54
|
+
role: 'judge',
|
|
55
|
+
system: JUDGE_SYSTEM,
|
|
56
|
+
prompt: [
|
|
57
|
+
'# What the developer asked', '', request.ask, '',
|
|
58
|
+
'# The agent\'s reply', '', reply, '',
|
|
59
|
+
'# A recommendation has to get these right', '', ...stage.required.map((r) => `- ${r}`), '',
|
|
60
|
+
'# True statements', '', ...ledger.facts.map((f) => `- ${f.claim}`), '',
|
|
61
|
+
'# Not true', '', ...ledger.notTrue.map((x) => `- ${x.claim}`),
|
|
62
|
+
].join('\n'),
|
|
63
|
+
schema: JUDGE_SCHEMA,
|
|
64
|
+
});
|
|
65
|
+
const g = (graded.structured ?? {});
|
|
66
|
+
const clean = { recommended: g.recommended === true, missing: g.missing ?? [], falseStatements: g.falseStatements ?? [] };
|
|
67
|
+
return { ...base, reply, ...clean, met: agentMet(request.fits, clean), note: g.note ?? '', costUsd: answer.costUsd + graded.costUsd };
|
|
68
|
+
}
|
|
69
|
+
catch (err) {
|
|
70
|
+
return { ...base, reply: '', recommended: false, missing: [], falseStatements: [], met: false, note: `error: ${err.message.slice(0, 200)}`, costUsd: 0 };
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/** Every request, several times each. Held-back requests are scored the same way and reported as a rate only. */
|
|
74
|
+
export async function agentStage(stage, pageText, ledger, llm, runsPerRequest, concurrency = 4) {
|
|
75
|
+
const jobs = stage.requests.flatMap((r) => Array.from({ length: runsPerRequest }, (_, i) => ({ r, n: i + 1 })));
|
|
76
|
+
const runs = [];
|
|
77
|
+
let next = 0;
|
|
78
|
+
async function worker() {
|
|
79
|
+
while (next < jobs.length) {
|
|
80
|
+
const job = jobs[next++];
|
|
81
|
+
runs.push(await once(stage, job.r, job.n, pageText, ledger, llm));
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, jobs.length) }, worker));
|
|
85
|
+
runs.sort((a, b) => a.request.localeCompare(b.request) || a.n - b.n);
|
|
86
|
+
const rate = (set) => wilson(set.filter((r) => r.met).length, set.length);
|
|
87
|
+
return { runs, working: rate(runs.filter((r) => !r.holdout)), holdout: rate(runs.filter((r) => r.holdout)), costUsd: runs.reduce((s, r) => s + r.costUsd, 0) };
|
|
88
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// Step 3: the owner approves the derived spec.
|
|
2
|
+
//
|
|
3
|
+
// Approval is of exact contents. It records a hash of everything the referee
|
|
4
|
+
// holds plus the claims ledger. If any of that changes afterwards, the project
|
|
5
|
+
// is no longer approved and nothing downstream may run until the owner
|
|
6
|
+
// approves again.
|
|
7
|
+
import { createHash } from 'node:crypto';
|
|
8
|
+
import { readdir, readFile, writeFile } from 'node:fs/promises';
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
const approvalFile = (l) => path.join(l.referee, 'approval.json');
|
|
11
|
+
async function filesUnder(dir) {
|
|
12
|
+
const out = [];
|
|
13
|
+
for (const entry of await readdir(dir, { withFileTypes: true }).catch(() => [])) {
|
|
14
|
+
const full = path.join(dir, entry.name);
|
|
15
|
+
if (entry.isDirectory())
|
|
16
|
+
out.push(...(await filesUnder(full)));
|
|
17
|
+
else
|
|
18
|
+
out.push(full);
|
|
19
|
+
}
|
|
20
|
+
return out;
|
|
21
|
+
}
|
|
22
|
+
/** A hash of the spec, every persona, and the ledger. The approval file itself is left out. */
|
|
23
|
+
export async function specHash(l) {
|
|
24
|
+
const files = [...(await filesUnder(l.referee)).filter((f) => f !== approvalFile(l)), l.factsFile].sort();
|
|
25
|
+
const hash = createHash('sha256');
|
|
26
|
+
for (const file of files) {
|
|
27
|
+
hash.update(path.relative(l.root, file));
|
|
28
|
+
hash.update('\0');
|
|
29
|
+
hash.update(await readFile(file));
|
|
30
|
+
hash.update('\0');
|
|
31
|
+
}
|
|
32
|
+
return hash.digest('hex');
|
|
33
|
+
}
|
|
34
|
+
export async function approve(l, options) {
|
|
35
|
+
if (!(options.budgetUsd > 0))
|
|
36
|
+
throw new Error('An approval needs a dollar budget for the simulations.');
|
|
37
|
+
await readFile(l.specFile).catch(() => { throw new Error('There is no derived spec in this project to approve.'); });
|
|
38
|
+
const approval = {
|
|
39
|
+
schema: 'sdd-approval/1',
|
|
40
|
+
hash: await specHash(l),
|
|
41
|
+
approvedAt: new Date().toISOString(),
|
|
42
|
+
budgetUsd: options.budgetUsd,
|
|
43
|
+
note: options.note?.trim() || undefined,
|
|
44
|
+
};
|
|
45
|
+
await writeFile(approvalFile(l), JSON.stringify(approval, null, 2) + '\n');
|
|
46
|
+
return approval;
|
|
47
|
+
}
|
|
48
|
+
/** Whether the spec as it stands now is the one the owner approved. */
|
|
49
|
+
export async function approvalState(l) {
|
|
50
|
+
let approval;
|
|
51
|
+
try {
|
|
52
|
+
approval = JSON.parse(await readFile(approvalFile(l), 'utf8'));
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return { approved: false, reason: 'The owner has not approved the spec.' };
|
|
56
|
+
}
|
|
57
|
+
if (approval.hash !== (await specHash(l))) {
|
|
58
|
+
return { approved: false, reason: 'The spec, the personas or the ledger changed after the owner approved them. It needs approving again.' };
|
|
59
|
+
}
|
|
60
|
+
return { approved: true, approval };
|
|
61
|
+
}
|
|
62
|
+
/** Throws unless the current spec is approved. Every later step calls this first. */
|
|
63
|
+
export async function requireApproval(l) {
|
|
64
|
+
const state = await approvalState(l);
|
|
65
|
+
if (!state.approved)
|
|
66
|
+
throw new Error(state.reason);
|
|
67
|
+
return state.approval;
|
|
68
|
+
}
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
// The pipeline's commands, called from engine/cli.ts.
|
|
2
|
+
//
|
|
3
|
+
// sdd derive "<result>" [--source <folder or address>]... [--budget-usd 5] [--project <id>]
|
|
4
|
+
// sdd show [<project>]
|
|
5
|
+
import { readdir, readFile, writeFile } from 'node:fs/promises';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import { parseArgs } from 'node:util';
|
|
8
|
+
import { createClaudeAgent } from "./agent.js";
|
|
9
|
+
import { approve, requireApproval } from "./approve.js";
|
|
10
|
+
import { DEFAULT_DERIVE_BUDGET_USD, derive } from "./derive.js";
|
|
11
|
+
import { designSpend, mockVerdict, runDesignLoop, scoreMock } from "./designloop.js";
|
|
12
|
+
import { parseLedger } from "./facts.js";
|
|
13
|
+
import { parsePipelineSpec } from "./spec.js";
|
|
14
|
+
import { createLLM } from "../runner/llm/index.js";
|
|
15
|
+
import { layout, logCost, projectId, projectsRoot } from "./project.js";
|
|
16
|
+
export const DERIVE_USAGE = 'Usage: sdd derive "<the result you want, in one sentence>" [--source <folder or address>]... [--budget-usd 5] [--project <id>]';
|
|
17
|
+
export async function deriveCommand(args, deps = {}) {
|
|
18
|
+
const log = deps.log ?? console.log;
|
|
19
|
+
const logError = deps.logError ?? console.error;
|
|
20
|
+
let parsed;
|
|
21
|
+
try {
|
|
22
|
+
parsed = parseArgs({
|
|
23
|
+
args,
|
|
24
|
+
allowPositionals: true,
|
|
25
|
+
options: {
|
|
26
|
+
source: { type: 'string', multiple: true },
|
|
27
|
+
'budget-usd': { type: 'string', default: String(DEFAULT_DERIVE_BUDGET_USD) },
|
|
28
|
+
project: { type: 'string' },
|
|
29
|
+
},
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
catch (err) {
|
|
33
|
+
logError(err.message);
|
|
34
|
+
logError(DERIVE_USAGE);
|
|
35
|
+
return 2;
|
|
36
|
+
}
|
|
37
|
+
const result = parsed.positionals.join(' ').trim();
|
|
38
|
+
const budgetUsd = Number(parsed.values['budget-usd']);
|
|
39
|
+
if (result.length < 10 || !(budgetUsd > 0)) {
|
|
40
|
+
logError(DERIVE_USAGE);
|
|
41
|
+
return 2;
|
|
42
|
+
}
|
|
43
|
+
// A folder is given to the deriver to read. Anything else is passed as it is, an address to look up.
|
|
44
|
+
const sources = (parsed.values.source ?? []).map((s) => (/^[a-z]+:\/\//i.test(s) ? s : path.resolve(s)));
|
|
45
|
+
const id = parsed.values.project ?? projectId(result);
|
|
46
|
+
const projectRoot = path.join(projectsRoot(deps.env), id);
|
|
47
|
+
log(`Project: ${id}`);
|
|
48
|
+
log(`Result: ${result}`);
|
|
49
|
+
try {
|
|
50
|
+
const outcome = await derive({ result, sources, projectRoot, budgetUsd, agent: deps.agent ?? createClaudeAgent(), log });
|
|
51
|
+
const working = outcome.spec.personas.filter((p) => !p.holdout).length;
|
|
52
|
+
log('');
|
|
53
|
+
log(`Derived: ${working} people and ${outcome.spec.personas.length - working} held back, ${outcome.spec.stages.length} stages, ${outcome.spec.assumptions.length} assumptions, ${outcome.ledger.facts.length} facts in the ledger.`);
|
|
54
|
+
log(`Cost at list prices: $${outcome.costUsd.toFixed(2)} (${outcome.attempts} attempt${outcome.attempts === 1 ? '' : 's'})`);
|
|
55
|
+
log(`Read it: ${outcome.layout.derivedSpec}`);
|
|
56
|
+
log('Nothing has been simulated or built. Approve the spec before anything else runs.');
|
|
57
|
+
return 0;
|
|
58
|
+
}
|
|
59
|
+
catch (err) {
|
|
60
|
+
logError(`Deriving failed: ${err.message}`);
|
|
61
|
+
return 1;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/** sdd revise <project> "<what the owner wants changed>" */
|
|
65
|
+
export async function reviseCommand(args, deps = {}) {
|
|
66
|
+
const log = deps.log ?? console.log;
|
|
67
|
+
const logError = deps.logError ?? console.error;
|
|
68
|
+
let parsed;
|
|
69
|
+
try {
|
|
70
|
+
parsed = parseArgs({ args, allowPositionals: true, options: { 'budget-usd': { type: 'string', default: String(DEFAULT_DERIVE_BUDGET_USD) }, file: { type: 'string' } } });
|
|
71
|
+
}
|
|
72
|
+
catch (err) {
|
|
73
|
+
logError(err.message);
|
|
74
|
+
return 2;
|
|
75
|
+
}
|
|
76
|
+
const [id, ...rest] = parsed.positionals;
|
|
77
|
+
const corrections = parsed.values.file ? await readFile(parsed.values.file, 'utf8').catch(() => '') : rest.join(' ');
|
|
78
|
+
const budgetUsd = Number(parsed.values['budget-usd']);
|
|
79
|
+
if (!id || corrections.trim().length < 10 || !(budgetUsd > 0)) {
|
|
80
|
+
logError('Usage: sdd revise <project> "<what to change>" [--file <corrections.md>] [--budget-usd 5]');
|
|
81
|
+
return 2;
|
|
82
|
+
}
|
|
83
|
+
const projectRoot = path.join(projectsRoot(deps.env), id);
|
|
84
|
+
const l = layout(projectRoot);
|
|
85
|
+
try {
|
|
86
|
+
const result = (await readFile(l.resultFile, 'utf8')).trim();
|
|
87
|
+
const sources = JSON.parse(await readFile(path.join(l.evidence, 'sources.json'), 'utf8').catch(() => '[]'));
|
|
88
|
+
const outcome = await derive({ result, sources, projectRoot, budgetUsd, corrections, agent: deps.agent ?? createClaudeAgent(), log });
|
|
89
|
+
log(`Revised. Cost at list prices: $${outcome.costUsd.toFixed(2)}. Any earlier approval no longer applies.`);
|
|
90
|
+
log(`Read it: ${outcome.layout.derivedSpec}`);
|
|
91
|
+
return 0;
|
|
92
|
+
}
|
|
93
|
+
catch (err) {
|
|
94
|
+
logError(`Revising failed: ${err.message}`);
|
|
95
|
+
return 1;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/** sdd approve <project> --budget-usd <n> [--note "..."]. Run by the owner, or on the owner's word. */
|
|
99
|
+
export async function approveCommand(args, deps = {}) {
|
|
100
|
+
const log = deps.log ?? console.log;
|
|
101
|
+
const logError = deps.logError ?? console.error;
|
|
102
|
+
let parsed;
|
|
103
|
+
try {
|
|
104
|
+
parsed = parseArgs({ args, allowPositionals: true, options: { 'budget-usd': { type: 'string' }, note: { type: 'string' } } });
|
|
105
|
+
}
|
|
106
|
+
catch (err) {
|
|
107
|
+
logError(err.message);
|
|
108
|
+
return 2;
|
|
109
|
+
}
|
|
110
|
+
const id = parsed.positionals[0];
|
|
111
|
+
const budgetUsd = Number(parsed.values['budget-usd']);
|
|
112
|
+
if (!id || !(budgetUsd > 0)) {
|
|
113
|
+
logError('Usage: sdd approve <project> --budget-usd <cap on simulation spend> [--note "..."]');
|
|
114
|
+
return 2;
|
|
115
|
+
}
|
|
116
|
+
try {
|
|
117
|
+
const approval = await approve(layout(path.join(projectsRoot(deps.env), id)), { budgetUsd, note: parsed.values.note });
|
|
118
|
+
log(`Approved ${approval.hash.slice(0, 12)} with a simulation budget of $${approval.budgetUsd.toFixed(2)}.`);
|
|
119
|
+
log('If the spec, the personas or the ledger change, this approval stops applying.');
|
|
120
|
+
return 0;
|
|
121
|
+
}
|
|
122
|
+
catch (err) {
|
|
123
|
+
logError(err.message);
|
|
124
|
+
return 1;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
/** sdd design <project> [--iterations 6] [--runs 4] [--try 5] */
|
|
128
|
+
export async function designCommand(args, deps = {}) {
|
|
129
|
+
const log = deps.log ?? console.log;
|
|
130
|
+
const logError = deps.logError ?? console.error;
|
|
131
|
+
let parsed;
|
|
132
|
+
try {
|
|
133
|
+
parsed = parseArgs({ args, allowPositionals: true, options: { iterations: { type: 'string', default: '6' }, runs: { type: 'string', default: '4' }, try: { type: 'string', default: '5' }, from: { type: 'string' }, rescore: { type: 'boolean', default: false } } });
|
|
134
|
+
}
|
|
135
|
+
catch (err) {
|
|
136
|
+
logError(err.message);
|
|
137
|
+
return 2;
|
|
138
|
+
}
|
|
139
|
+
const id = parsed.positionals[0];
|
|
140
|
+
if (!id) {
|
|
141
|
+
logError('Usage: sdd design <project> [--iterations 6] [--runs 4] [--try 5] [--from <design number> [--rescore]]');
|
|
142
|
+
return 2;
|
|
143
|
+
}
|
|
144
|
+
const l = layout(path.join(projectsRoot(deps.env), id));
|
|
145
|
+
try {
|
|
146
|
+
const spec = parsePipelineSpec(JSON.parse(await readFile(l.specFile, 'utf8')));
|
|
147
|
+
const ledger = parseLedger(JSON.parse(await readFile(l.factsFile, 'utf8')));
|
|
148
|
+
// --from <n>: carry on from design n. New designs are numbered after the last one already made.
|
|
149
|
+
const resumeFrom = parsed.values.from === undefined ? undefined : Number(parsed.values.from);
|
|
150
|
+
const made = (await readdir(l.design).catch(() => [])).map((d) => Number(d.match(/^iter-(\d+)$/)?.[1])).filter((n) => !Number.isNaN(n));
|
|
151
|
+
const startAt = made.length > 0 ? Math.max(...made) + 1 : 0;
|
|
152
|
+
// The owner's own places and the repository folders, for the guardrails.
|
|
153
|
+
const rules = await readFile(path.join(path.dirname(l.referee), 'rules.json'), 'utf8').then((t) => JSON.parse(t), () => ({}));
|
|
154
|
+
const sources = await readFile(path.join(l.evidence, 'sources.json'), 'utf8').then((t) => JSON.parse(t), () => []);
|
|
155
|
+
const sourceDirs = sources.filter((x) => !/^https?:/.test(x)).map((x) => path.resolve(x.replace(/^~(?=\/)/, process.env.HOME ?? '~')));
|
|
156
|
+
const result = await runDesignLoop({
|
|
157
|
+
agent: deps.agent ?? createClaudeAgent(), sourceDirs, ownPlaces: rules.ownPlaces ?? [],
|
|
158
|
+
layout: l, spec, ledger, llm: deps.llm ?? createLLM(), log,
|
|
159
|
+
maxIterations: Number(parsed.values.iterations), runs: Number(parsed.values.runs), tryCount: Number(parsed.values.try),
|
|
160
|
+
...(resumeFrom === undefined ? {} : { resumeFrom, startAt, rescoreFirst: parsed.values.rescore }),
|
|
161
|
+
});
|
|
162
|
+
await writeFile(path.join(l.evidence, resumeFrom === undefined ? 'design-loop.json' : `design-loop-from-${resumeFrom}.json`), JSON.stringify({ ...result, best: result.best && { iteration: result.best.iteration, verdict: result.best.verdict, dir: result.best.dir } }, null, 2) + '\n');
|
|
163
|
+
log('');
|
|
164
|
+
log(`Stopped: ${result.stop}. Spent $${result.spentUsd.toFixed(2)} at list prices.`);
|
|
165
|
+
for (const it of result.iterations)
|
|
166
|
+
log(` design ${it.iteration}: ${it.outcome}${it.verdict ? `, ${it.verdict}, shortfall ${it.shortfall?.toFixed(2)}` : ''}`);
|
|
167
|
+
if (result.best)
|
|
168
|
+
log(`Best: design ${result.best.iteration} (${result.best.verdict}) at ${result.best.dir}`);
|
|
169
|
+
return 0;
|
|
170
|
+
}
|
|
171
|
+
catch (err) {
|
|
172
|
+
logError(`Design failed: ${err.message}`);
|
|
173
|
+
return 1;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Score a page that the pipeline did not design, such as the site as it stands
|
|
178
|
+
* today, with the same people, questions and clean machines. The page is
|
|
179
|
+
* served from a local folder. It is a reference point and never feeds the designer.
|
|
180
|
+
*/
|
|
181
|
+
export async function scoreCommand(args, deps = {}) {
|
|
182
|
+
const log = deps.log ?? console.log;
|
|
183
|
+
const logError = deps.logError ?? console.error;
|
|
184
|
+
const usage = 'Usage: sdd score <project> (--page <folder with index.html> | --target http://localhost:PORT [--entry /path]) --name <label> [--runs 4] [--try 5] [--load-seconds n] [--only read,decide,try,agent]';
|
|
185
|
+
let parsed;
|
|
186
|
+
try {
|
|
187
|
+
parsed = parseArgs({ args, allowPositionals: true, options: { page: { type: 'string' }, name: { type: 'string' }, runs: { type: 'string', default: '4' }, try: { type: 'string', default: '5' }, 'load-seconds': { type: 'string' }, only: { type: 'string' }, target: { type: 'string' }, entry: { type: 'string', default: '/' } } });
|
|
188
|
+
}
|
|
189
|
+
catch (err) {
|
|
190
|
+
logError(err.message);
|
|
191
|
+
return 2;
|
|
192
|
+
}
|
|
193
|
+
const id = parsed.positionals[0];
|
|
194
|
+
const { name, target } = parsed.values;
|
|
195
|
+
// A running build on this machine can stand in for a folder: the verify step scores the real thing.
|
|
196
|
+
if (target && !/^http:\/\/(localhost|127\.0\.0\.1)(:\d+)?$/.test(target)) {
|
|
197
|
+
logError('--target must be a build running on this machine, such as http://localhost:3000. Live sites are never scored.');
|
|
198
|
+
return 2;
|
|
199
|
+
}
|
|
200
|
+
const page = parsed.values.page ?? (target ? '.' : undefined);
|
|
201
|
+
if (!id || !page || !name || !/^[a-z0-9][a-z0-9-]*$/.test(name)) {
|
|
202
|
+
logError(usage);
|
|
203
|
+
return 2;
|
|
204
|
+
}
|
|
205
|
+
const l = layout(path.join(projectsRoot(deps.env), id));
|
|
206
|
+
try {
|
|
207
|
+
const approval = await requireApproval(l);
|
|
208
|
+
const spec = parsePipelineSpec(JSON.parse(await readFile(l.specFile, 'utf8')));
|
|
209
|
+
const left = approval.budgetUsd - (await designSpend(l));
|
|
210
|
+
if (left <= 0.5) {
|
|
211
|
+
logError('The approved budget is spent.');
|
|
212
|
+
return 3;
|
|
213
|
+
}
|
|
214
|
+
const runs = Number(parsed.values.runs);
|
|
215
|
+
const tryCount = Number(parsed.values.try);
|
|
216
|
+
const evidenceDir = path.join(l.evidence, 'score', name);
|
|
217
|
+
const ledger = parseLedger(JSON.parse(await readFile(l.factsFile, 'utf8')));
|
|
218
|
+
// For sweeping an assumption: a slower page, or only some stages.
|
|
219
|
+
const loadSeconds = parsed.values['load-seconds'] === undefined ? undefined : Number(parsed.values['load-seconds']);
|
|
220
|
+
const only = parsed.values.only?.split(',').map((x) => x.trim());
|
|
221
|
+
const score = await scoreMock({ layout: l, spec, mockDir: path.resolve(page), evidenceDir, runs, tryCount, budgetUsd: left, llm: deps.llm ?? createLLM(), ledger, log, loadSeconds, only, running: target ? { origin: target, entry: parsed.values.entry } : undefined });
|
|
222
|
+
await logCost(l, { step: 'design', what: `simulation, page "${name}"`, usd: score.costUsd, durationMs: 0, detail: { runs, tryCount } });
|
|
223
|
+
const verdict = mockVerdict(score);
|
|
224
|
+
await writeFile(path.join(evidenceDir, 'score.json'), JSON.stringify({ name, page: path.resolve(page), runs, tryCount, verdict, score }, null, 2) + '\n');
|
|
225
|
+
log('');
|
|
226
|
+
log(`${name}: ${verdict}. Cost $${score.costUsd.toFixed(2)} at list prices.`);
|
|
227
|
+
for (const q of score.questions)
|
|
228
|
+
log(` ${q.stage}/${q.id}: seen ${q.working.passed}/${q.working.n}, held back ${q.holdout.passed}/${q.holdout.n}, needs ${Math.round(q.threshold * 100)}%`);
|
|
229
|
+
if (score.tried)
|
|
230
|
+
log(` ${score.tried.stage}: ${score.tried.interval.passed}/${score.tried.interval.n}, needs ${Math.round(score.tried.threshold * 100)}%`);
|
|
231
|
+
for (const n of score.notRun)
|
|
232
|
+
log(` ${n.stage}: not run (${n.why})`);
|
|
233
|
+
return 0;
|
|
234
|
+
}
|
|
235
|
+
catch (err) {
|
|
236
|
+
logError(`Scoring failed: ${err.message}`);
|
|
237
|
+
return 1;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
export async function showCommand(args, deps = {}) {
|
|
241
|
+
const log = deps.log ?? console.log;
|
|
242
|
+
const logError = deps.logError ?? console.error;
|
|
243
|
+
const root = projectsRoot(deps.env);
|
|
244
|
+
const id = args[0];
|
|
245
|
+
if (!id) {
|
|
246
|
+
const ids = (await readdir(root).catch(() => [])).sort();
|
|
247
|
+
if (ids.length === 0)
|
|
248
|
+
log('No projects yet. Start one with: sdd derive "<the result you want>"');
|
|
249
|
+
for (const name of ids)
|
|
250
|
+
log(name);
|
|
251
|
+
return 0;
|
|
252
|
+
}
|
|
253
|
+
try {
|
|
254
|
+
log(await readFile(layout(path.join(root, id)).derivedSpec, 'utf8'));
|
|
255
|
+
return 0;
|
|
256
|
+
}
|
|
257
|
+
catch {
|
|
258
|
+
logError(`No derived spec for project "${id}" under ${root}`);
|
|
259
|
+
return 1;
|
|
260
|
+
}
|
|
261
|
+
}
|