sdd-engine 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +178 -2
  3. package/dist/engine/cli.js +95 -0
  4. package/dist/engine/simulate.js +214 -0
  5. package/dist/judge/conversation.js +45 -0
  6. package/dist/judge/judge.js +122 -0
  7. package/dist/judge/outcome.js +42 -0
  8. package/dist/judge/regrade.js +74 -0
  9. package/dist/pipeline/agent.js +78 -0
  10. package/dist/pipeline/agentstage.js +88 -0
  11. package/dist/pipeline/approve.js +68 -0
  12. package/dist/pipeline/cli.js +261 -0
  13. package/dist/pipeline/derive.js +287 -0
  14. package/dist/pipeline/designloop.js +385 -0
  15. package/dist/pipeline/facts.js +108 -0
  16. package/dist/pipeline/mock.js +253 -0
  17. package/dist/pipeline/project.js +79 -0
  18. package/dist/pipeline/render.js +95 -0
  19. package/dist/pipeline/spec.js +272 -0
  20. package/dist/pipeline/try.js +162 -0
  21. package/dist/runner/batch-lib.js +157 -0
  22. package/dist/runner/batch.js +56 -0
  23. package/dist/runner/cli.js +61 -0
  24. package/dist/runner/llm/claude-code.js +123 -0
  25. package/dist/runner/llm/index.js +12 -0
  26. package/dist/runner/llm/types.js +17 -0
  27. package/dist/runner/reader/answer.js +76 -0
  28. package/dist/runner/reader/extract.js +106 -0
  29. package/dist/runner/reader/run.js +111 -0
  30. package/dist/runner/reader/skim.js +115 -0
  31. package/dist/runner/rng.js +22 -0
  32. package/dist/runner/sandbox/cli.js +60 -0
  33. package/dist/runner/sandbox/sandbox.js +144 -0
  34. package/dist/runner/sandbox/server.js +64 -0
  35. package/dist/runner/spec.js +84 -0
  36. package/dist/runner/talker/decide.js +110 -0
  37. package/dist/runner/talker/run.js +159 -0
  38. package/dist/runner/talker/world.js +36 -0
  39. package/dist/runner/trace.js +130 -0
  40. package/dist/runner/visitor/decide.js +159 -0
  41. package/dist/runner/visitor/observe.js +102 -0
  42. package/dist/runner/visitor/run.js +296 -0
  43. package/dist/scorer/compare.js +119 -0
  44. package/dist/scorer/score.js +201 -0
  45. package/dist/scorer/spot-check-compare.js +41 -0
  46. package/dist/scorer/spot-check-view.js +45 -0
  47. package/dist/scorer/stats.js +45 -0
  48. package/dist/scorer/summary.js +62 -0
  49. package/package.json +49 -4
@@ -0,0 +1,122 @@
1
+ // The judge. Part of the referee: the revising agent may never edit this.
2
+ //
3
+ // It sees the questions, the answers, and the rubric. Nothing else. It does
4
+ // not get the page, the persona, the viewport, the seed, or anything that
5
+ // says which page variant produced the answers.
6
+ //
7
+ // Version 2. Version 1 gave a verdict first and a note after, and a blind
8
+ // second review agreed with it on only 64% of contrast answers. Its notes
9
+ // sometimes contradicted its own verdicts. Version 2 must quote the evidence
10
+ // for each element before it decides.
11
+ export const JUDGE_VERSION = 'v2';
12
+ const SYSTEM = `You grade short written answers against a rubric. You are careful and consistent.
13
+
14
+ For each question you get the question, one person's answer, and a list of rubric elements. For each element, first quote the words from the answer that bear on it, then decide whether the answer meets it.
15
+
16
+ Rules:
17
+ - Evidence first. Copy the relevant words from the answer exactly. If there are none, write "none". Decide only after quoting.
18
+ - Grade the meaning, not the wording. A clear paraphrase meets an element.
19
+ - An element with several parts is met only if every part is stated. For example "a test checks whether code is correct, pass or fail" needs both that a test checks the code or its correctness, and that the result is binary (pass or fail, passing or failing, or the like). An example assertion on its own does not state a binary result.
20
+ - A part can be stated by contrast. "A score against a threshold rather than a pass or fail" states that the other side is pass or fail.
21
+ - Do not give credit for what the person probably meant, and do not use your own knowledge of the subject to fill gaps.
22
+ - People hedge. A claim counts if the person states it, even if they add that they are unsure about other details or about how the page framed it. A claim does not count if the person says they do not know it or cannot remember it.
23
+ - An element that asks for "at least N of" a list is met if at least N of the listed items are named, in any wording or letter case. Being unsure of their order does not matter.
24
+ - Grade each question on its own answer only. Do not carry information from one answer to another.
25
+ - The note is one short sentence saying what was missing, or "complete". It must agree with your decisions.`;
26
+ export function buildJudgeRequest(questions, answers) {
27
+ const body = questions
28
+ .map((q) => {
29
+ const answer = answers.find((a) => a.id === q.id);
30
+ if (!answer)
31
+ throw new Error(`No answer to grade for question "${q.id}"`);
32
+ const required = q.rubric.required.map((r, i) => ` required ${i + 1}: ${r}`).join('\n');
33
+ const bonus = (q.rubric.bonus ?? []).map((r, i) => ` bonus ${i + 1}: ${r}`).join('\n');
34
+ return `QUESTION ${q.id}: ${q.ask}\nANSWER: ${answer.answer}\nRUBRIC:\n${required}${bonus ? `\n${bonus}` : ''}`;
35
+ })
36
+ .join('\n\n');
37
+ const element = {
38
+ type: 'object',
39
+ properties: { evidence: { type: 'string' }, met: { type: 'boolean' } },
40
+ required: ['evidence', 'met'],
41
+ };
42
+ return {
43
+ role: 'judge',
44
+ system: SYSTEM,
45
+ prompt: `${body}\n\nGrade every question by its id: ${questions.map((q) => q.id).join(', ')}. Return exactly ${questions.length} grades. For each, give one entry per rubric element, in the order listed.`,
46
+ schema: {
47
+ type: 'object',
48
+ properties: {
49
+ grades: {
50
+ type: 'array',
51
+ minItems: questions.length,
52
+ maxItems: questions.length,
53
+ items: {
54
+ type: 'object',
55
+ properties: {
56
+ id: { type: 'string' },
57
+ required: { type: 'array', items: element },
58
+ bonus: { type: 'array', items: element },
59
+ note: { type: 'string' },
60
+ },
61
+ required: ['id', 'required', 'bonus', 'note'],
62
+ },
63
+ },
64
+ },
65
+ required: ['grades'],
66
+ },
67
+ };
68
+ }
69
+ export async function judgeAnswers(llm, questions, answers) {
70
+ const request = buildJudgeRequest(questions, answers);
71
+ const calls = [];
72
+ let problem = '';
73
+ // The judge sometimes returns grades for only some of the questions. Ask
74
+ // again with the same request, never a changed one. Every attempt is kept
75
+ // and paid for.
76
+ for (let attempt = 0; attempt < 3; attempt++) {
77
+ const response = await llm.complete(request);
78
+ calls.push({ request, response });
79
+ const reply = response.structured;
80
+ const grades = [];
81
+ problem = '';
82
+ for (const q of questions) {
83
+ const g = reply?.grades?.find((x) => x.id === q.id);
84
+ const bonusCount = q.rubric.bonus?.length ?? 0;
85
+ const wellFormed = g &&
86
+ g.required?.length === q.rubric.required.length &&
87
+ (g.bonus?.length ?? 0) >= bonusCount &&
88
+ [...g.required, ...g.bonus].every((e) => typeof e?.met === 'boolean');
89
+ if (!g || !wellFormed) {
90
+ problem = `Judge reply is missing or malformed for question "${q.id}". Full reply: ${JSON.stringify(reply).slice(0, 600)}`;
91
+ break;
92
+ }
93
+ grades.push({
94
+ questionId: q.id,
95
+ pass: g.required.every((e) => e.met), // a pass needs every required element
96
+ requiredMet: g.required.map((e) => e.met),
97
+ bonusMet: g.bonus.slice(0, bonusCount).map((e) => e.met),
98
+ evidence: g.required.map((e) => e.evidence),
99
+ note: g.note,
100
+ });
101
+ }
102
+ if (!problem)
103
+ return { grades, calls };
104
+ }
105
+ throw new Error(problem);
106
+ }
107
+ /** Returns a graded copy of a not yet stored trace, with the judge calls and their cost added. */
108
+ export async function gradeTrace(llm, questions, trace) {
109
+ const { grades, calls } = await judgeAnswers(llm, questions, trace.answers);
110
+ const llmCalls = [...trace.llmCalls, ...calls];
111
+ const tokens = (c) => c.response.usage.inputTokens + c.response.usage.cacheReadTokens + c.response.usage.cacheWriteTokens;
112
+ return {
113
+ ...trace,
114
+ grades,
115
+ llmCalls,
116
+ cost: {
117
+ usd: llmCalls.reduce((sum, c) => sum + c.response.costUsd, 0),
118
+ inputTokens: llmCalls.reduce((sum, c) => sum + tokens(c), 0),
119
+ outputTokens: llmCalls.reduce((sum, c) => sum + c.response.usage.outputTokens, 0),
120
+ },
121
+ };
122
+ }
@@ -0,0 +1,42 @@
1
+ // Judges whether a visitor's session ended with the expected outcome on
2
+ // screen. Part of the referee.
3
+ //
4
+ // Used when a task's success cannot be checked mechanically (no known
5
+ // endpoint, URL, or exact text), which is the usual case for an app built
6
+ // from a prose spec: "the new plant appears in the list with its next
7
+ // watering date". The judge sees the expected outcome and the final screen.
8
+ // It does not see the persona, the visitor's reasoning, or the steps taken.
9
+ const SYSTEM = `You check whether a web app's screen shows that a task was actually completed. You are strict.
10
+
11
+ You get the expected outcome and the text on the screen where a visitor's session ended. Decide whether the screen shows the expected outcome.
12
+
13
+ Rules:
14
+ - Evidence first. Quote the exact words on the screen that show the outcome. If there are none, write "none". Decide only after quoting.
15
+ - The outcome must be visible on this screen. A form that is filled in but not submitted is not the outcome. A button that would produce the outcome is not the outcome.
16
+ - An error message, a validation message, or a login wall means the outcome is not met.
17
+ - Do not give credit for what probably happened. Judge only what the screen shows.
18
+ - The note is one short sentence: what shows the outcome, or what is missing.`;
19
+ export function buildOutcomeRequest(expected, screen) {
20
+ return {
21
+ role: 'judge',
22
+ system: SYSTEM,
23
+ prompt: `EXPECTED OUTCOME\n${expected}\n\nFINAL SCREEN\nPage: "${screen.title}" at ${screen.url}\n${screen.text.length ? screen.text.map((t) => `- ${t}`).join('\n') : '(no text on screen)'}\n\nDoes the final screen show the expected outcome?`,
24
+ schema: {
25
+ type: 'object',
26
+ properties: { evidence: { type: 'string' }, met: { type: 'boolean' }, note: { type: 'string' } },
27
+ required: ['evidence', 'met', 'note'],
28
+ },
29
+ };
30
+ }
31
+ export async function judgeOutcome(llm, expected, screen) {
32
+ const request = buildOutcomeRequest(expected, screen);
33
+ const calls = [];
34
+ for (let attempt = 0; attempt < 2; attempt++) {
35
+ const response = await llm.complete(request);
36
+ calls.push({ request, response });
37
+ const reply = response.structured;
38
+ if (reply && typeof reply.met === 'boolean')
39
+ return { met: reply.met, evidence: reply.evidence ?? '', note: reply.note ?? '', calls };
40
+ }
41
+ throw new Error('The outcome judge gave no verdict');
42
+ }
@@ -0,0 +1,74 @@
1
+ // Regrades the stored answers of a batch with the current judge.
2
+ //
3
+ // node judge/regrade.ts --batch <id> --label <name> --budget-usd <n> [--concurrency 6]
4
+ //
5
+ // Traces are never edited. Each regrade is written next to them, to
6
+ // grades/<experiment>/<batch>/<label>/<runId>.json, with the judge's full
7
+ // prompts and replies. The scorer reads them with --grades <label>.
8
+ // Safe to rerun: runs already regraded under the label are skipped.
9
+ import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
10
+ import path from 'node:path';
11
+ import { parseArgs } from 'node:util';
12
+ import { createLLM } from "../runner/llm/index.js";
13
+ import { loadSpec } from "../runner/spec.js";
14
+ import { JUDGE_VERSION, judgeAnswers } from "./judge.js";
15
+ const repoRoot = path.resolve(import.meta.dirname, '..');
16
+ const { values } = parseArgs({
17
+ options: {
18
+ spec: { type: 'string', default: 'specs/sdd-site.spec.yaml' },
19
+ batch: { type: 'string' },
20
+ label: { type: 'string' },
21
+ concurrency: { type: 'string', default: '6' },
22
+ 'budget-usd': { type: 'string' },
23
+ },
24
+ });
25
+ if (!values.batch || !values.label || !values['budget-usd']) {
26
+ console.error('Usage: node judge/regrade.ts --batch <id> --label <name> --budget-usd <n> [--concurrency n]');
27
+ process.exit(2);
28
+ }
29
+ const budgetUsd = Number(values['budget-usd']);
30
+ const spec = await loadSpec(path.resolve(repoRoot, values.spec));
31
+ const tracesDir = path.join(repoRoot, 'traces', spec.experiment);
32
+ const outDir = path.join(repoRoot, 'grades', spec.experiment, values.batch, values.label);
33
+ await mkdir(outDir, { recursive: true });
34
+ const done = new Set(await readdir(outDir));
35
+ const todo = [];
36
+ for (const file of (await readdir(tracesDir)).sort()) {
37
+ if (!file.endsWith('.json') || done.has(file))
38
+ continue;
39
+ const trace = JSON.parse(await readFile(path.join(tracesDir, file), 'utf8'));
40
+ if (trace.batch === values.batch)
41
+ todo.push(trace);
42
+ }
43
+ console.log(`Regrade "${values.label}" of batch "${values.batch}" with judge ${JUDGE_VERSION}: ${done.size} done, ${todo.length} to do`);
44
+ const llm = createLLM();
45
+ let next = 0, finished = 0, failed = 0, spent = 0, budgetHit = false;
46
+ async function worker() {
47
+ while (next < todo.length) {
48
+ if (spent >= budgetUsd) {
49
+ budgetHit = true;
50
+ return;
51
+ }
52
+ const trace = todo[next++];
53
+ try {
54
+ const { grades, calls } = await judgeAnswers(llm, spec.questions, trace.answers);
55
+ const costUsd = calls.reduce((s, c) => s + c.response.costUsd, 0);
56
+ spent += costUsd;
57
+ await writeFile(path.join(outDir, `${trace.runId}.json`), JSON.stringify({ runId: trace.runId, batch: trace.batch, label: values.label, judgeVersion: JUDGE_VERSION, grades, llmCalls: calls, costUsd }, null, 2) + '\n', { flag: 'wx' });
58
+ }
59
+ catch (err) {
60
+ failed++;
61
+ console.error(`FAILED ${trace.runId}: ${String(err).slice(0, 500)}`);
62
+ }
63
+ if (++finished % 40 === 0 || finished === todo.length)
64
+ console.log(` ${finished}/${todo.length}, ${failed} failed, $${spent.toFixed(2)}`);
65
+ }
66
+ }
67
+ await Promise.all(Array.from({ length: Number(values.concurrency) }, worker));
68
+ console.log(`Done: ${finished - failed} regraded, ${failed} failed, $${spent.toFixed(2)} spent`);
69
+ if (budgetHit) {
70
+ console.error(`STOPPED: budget of $${budgetUsd.toFixed(2)} reached.`);
71
+ process.exit(3);
72
+ }
73
+ if (failed > 0)
74
+ process.exit(1);
@@ -0,0 +1,78 @@
1
+ // An agent session with tools, for pipeline steps that have to look and
2
+ // measure. The engine's own model calls have no tools at all (see
3
+ // runner/llm/claude-code.ts); this is the other kind.
4
+ //
5
+ // Every session has a named list of tools, a list of folders it may read
6
+ // beyond its own, and a dollar cap. Its cost is reported back for the log.
7
+ import { spawn } from 'node:child_process';
8
+ export const DEFAULT_AGENT_MODEL = 'claude-opus-5-5';
9
+ const DEFAULT_TIMEOUT_MS = 30 * 60 * 1000;
10
+ export function agentArgs(options, env = process.env) {
11
+ return [
12
+ '-p',
13
+ '--model', options.model ?? env.SDD_MODEL_DERIVER ?? DEFAULT_AGENT_MODEL,
14
+ '--output-format', 'json',
15
+ '--max-budget-usd', options.budgetUsd.toFixed(2),
16
+ // Only these tools exist for the session, and they run without asking.
17
+ '--tools', options.tools.join(','),
18
+ '--allowedTools', options.tools.join(','),
19
+ // No user settings, no MCP servers: it must not reach a memory of earlier conversations.
20
+ '--strict-mcp-config',
21
+ '--setting-sources', '',
22
+ '--disable-slash-commands',
23
+ '--no-session-persistence',
24
+ ...options.addDirs.flatMap((dir) => ['--add-dir', dir]),
25
+ ];
26
+ }
27
+ /** Reads the JSON the CLI prints in print mode. */
28
+ export function parseAgentOutput(stdout, exitCode, stderr) {
29
+ try {
30
+ const out = JSON.parse(stdout);
31
+ const ok = exitCode === 0 && out.is_error !== true;
32
+ return {
33
+ ok,
34
+ text: out.result ?? '',
35
+ costUsd: typeof out.total_cost_usd === 'number' ? out.total_cost_usd : 0,
36
+ turns: out.num_turns,
37
+ durationMs: out.duration_ms,
38
+ error: ok ? undefined : `${out.subtype ?? 'error'}: ${(out.result ?? stderr).slice(0, 600)}`,
39
+ };
40
+ }
41
+ catch {
42
+ return { ok: false, text: '', costUsd: 0, error: `claude exited ${exitCode}: ${(stderr || stdout).slice(0, 600)}` };
43
+ }
44
+ }
45
+ export function createClaudeAgent() {
46
+ return {
47
+ run(options) {
48
+ const started = Date.now();
49
+ return new Promise((resolve) => {
50
+ const child = spawn('claude', agentArgs(options), { cwd: options.cwd, stdio: ['pipe', 'pipe', 'pipe'] });
51
+ let out = '';
52
+ let err = '';
53
+ let timedOut = false;
54
+ const timer = setTimeout(() => {
55
+ timedOut = true;
56
+ child.kill('SIGTERM');
57
+ }, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
58
+ child.stdout.on('data', (d) => (out += d));
59
+ child.stderr.on('data', (d) => (err += d));
60
+ child.on('error', (e) => {
61
+ clearTimeout(timer);
62
+ resolve({ ok: false, text: '', costUsd: 0, durationMs: Date.now() - started, error: `could not start claude: ${e.message}` });
63
+ });
64
+ child.on('close', (code) => {
65
+ clearTimeout(timer);
66
+ const parsed = parseAgentOutput(out, code, err);
67
+ resolve({
68
+ ...parsed,
69
+ durationMs: Date.now() - started,
70
+ ok: parsed.ok && !timedOut,
71
+ error: timedOut ? `${options.name} was stopped after ${Math.round((options.timeoutMs ?? DEFAULT_TIMEOUT_MS) / 60000)} minutes` : parsed.error,
72
+ });
73
+ });
74
+ child.stdin.end(options.prompt);
75
+ });
76
+ },
77
+ };
78
+ }
@@ -0,0 +1,88 @@
1
+ // The "agent" stage: the visitor is an AI coding agent, not a person.
2
+ //
3
+ // A developer asks their agent for something. The agent fetches the page, reads
4
+ // all of its text (no layout, no images, no time limit) and replies to its
5
+ // developer. A judge then reads the reply: did it recommend the product for
6
+ // this request, did it state what a recommendation has to get right, and did
7
+ // it say anything the ledger does not support.
8
+ //
9
+ // Some requests are ones the product does not fit. For those the right reply
10
+ // is not to recommend it, so a page that oversells scores worse, not better.
11
+ import { wilson } from "../scorer/stats.js";
12
+ const AGENT_SYSTEM = `You are an AI coding agent working for a developer in their terminal. Your developer asked you for something. You searched and fetched one web page. Its text is below.
13
+
14
+ Reply to your developer the way you would in the session, in a few sentences:
15
+ - Say whether this is what they asked for.
16
+ - If it is, say whether they should try it, exactly what to run, and what they need in place first.
17
+ - If it is not a fit for what they asked, say so plainly and do not recommend it for that.
18
+ Use only what the page says about the product. Do not add anything from memory, and do not guess at what the page leaves out: if something they need to know is missing, say it is not stated.`;
19
+ const JUDGE_SYSTEM = `You grade one reply an AI coding agent gave its developer after reading a product's web page.
20
+
21
+ You get what the developer asked, the agent's reply, the points a recommendation has to get right, and a ledger of true statements about the product with a list of statements that are not true.
22
+ - recommended: true only if the reply tells the developer to try or use the product for what they asked. "It could work but I would not use it for this" is false.
23
+ - missing: each required point the reply does not state. Rewording is fine. A command must be given as written.
24
+ - falseStatements: anything the reply says about the product that the ledger does not support or that matches the "not true" list. Quote the reply. Saying that something is not stated on the page is not a false statement.
25
+ Grade the reply only. Do not reward or punish style.`;
26
+ const JUDGE_SCHEMA = {
27
+ type: 'object',
28
+ properties: {
29
+ recommended: { type: 'boolean' },
30
+ missing: { type: 'array', items: { type: 'string' } },
31
+ falseStatements: { type: 'array', items: { type: 'string' } },
32
+ note: { type: 'string', description: 'One line on why.' },
33
+ },
34
+ required: ['recommended', 'missing', 'falseStatements', 'note'],
35
+ additionalProperties: false,
36
+ };
37
+ /** Whether one graded reply counts. Decided by code from what the judge reported. */
38
+ export function agentMet(fits, graded) {
39
+ if (!fits)
40
+ return !graded.recommended;
41
+ return graded.recommended && graded.missing.length === 0 && graded.falseStatements.length === 0;
42
+ }
43
+ async function once(stage, request, n, pageText, ledger, llm) {
44
+ const base = { request: request.id, holdout: request.holdout, fits: request.fits, n };
45
+ try {
46
+ const answer = await llm.complete({
47
+ role: 'persona',
48
+ system: AGENT_SYSTEM,
49
+ prompt: [`# What your developer asked`, '', request.ask, '', '# The page you fetched', '', pageText].join('\n'),
50
+ schema: { type: 'object', properties: { reply: { type: 'string' } }, required: ['reply'], additionalProperties: false },
51
+ });
52
+ const reply = answer.structured?.reply ?? answer.text;
53
+ const graded = await llm.complete({
54
+ role: 'judge',
55
+ system: JUDGE_SYSTEM,
56
+ prompt: [
57
+ '# What the developer asked', '', request.ask, '',
58
+ '# The agent\'s reply', '', reply, '',
59
+ '# A recommendation has to get these right', '', ...stage.required.map((r) => `- ${r}`), '',
60
+ '# True statements', '', ...ledger.facts.map((f) => `- ${f.claim}`), '',
61
+ '# Not true', '', ...ledger.notTrue.map((x) => `- ${x.claim}`),
62
+ ].join('\n'),
63
+ schema: JUDGE_SCHEMA,
64
+ });
65
+ const g = (graded.structured ?? {});
66
+ const clean = { recommended: g.recommended === true, missing: g.missing ?? [], falseStatements: g.falseStatements ?? [] };
67
+ return { ...base, reply, ...clean, met: agentMet(request.fits, clean), note: g.note ?? '', costUsd: answer.costUsd + graded.costUsd };
68
+ }
69
+ catch (err) {
70
+ return { ...base, reply: '', recommended: false, missing: [], falseStatements: [], met: false, note: `error: ${err.message.slice(0, 200)}`, costUsd: 0 };
71
+ }
72
+ }
73
+ /** Every request, several times each. Held-back requests are scored the same way and reported as a rate only. */
74
+ export async function agentStage(stage, pageText, ledger, llm, runsPerRequest, concurrency = 4) {
75
+ const jobs = stage.requests.flatMap((r) => Array.from({ length: runsPerRequest }, (_, i) => ({ r, n: i + 1 })));
76
+ const runs = [];
77
+ let next = 0;
78
+ async function worker() {
79
+ while (next < jobs.length) {
80
+ const job = jobs[next++];
81
+ runs.push(await once(stage, job.r, job.n, pageText, ledger, llm));
82
+ }
83
+ }
84
+ await Promise.all(Array.from({ length: Math.min(concurrency, jobs.length) }, worker));
85
+ runs.sort((a, b) => a.request.localeCompare(b.request) || a.n - b.n);
86
+ const rate = (set) => wilson(set.filter((r) => r.met).length, set.length);
87
+ return { runs, working: rate(runs.filter((r) => !r.holdout)), holdout: rate(runs.filter((r) => r.holdout)), costUsd: runs.reduce((s, r) => s + r.costUsd, 0) };
88
+ }
@@ -0,0 +1,68 @@
1
+ // Step 3: the owner approves the derived spec.
2
+ //
3
+ // Approval is of exact contents. It records a hash of everything the referee
4
+ // holds plus the claims ledger. If any of that changes afterwards, the project
5
+ // is no longer approved and nothing downstream may run until the owner
6
+ // approves again.
7
+ import { createHash } from 'node:crypto';
8
+ import { readdir, readFile, writeFile } from 'node:fs/promises';
9
+ import path from 'node:path';
10
+ const approvalFile = (l) => path.join(l.referee, 'approval.json');
11
+ async function filesUnder(dir) {
12
+ const out = [];
13
+ for (const entry of await readdir(dir, { withFileTypes: true }).catch(() => [])) {
14
+ const full = path.join(dir, entry.name);
15
+ if (entry.isDirectory())
16
+ out.push(...(await filesUnder(full)));
17
+ else
18
+ out.push(full);
19
+ }
20
+ return out;
21
+ }
22
+ /** A hash of the spec, every persona, and the ledger. The approval file itself is left out. */
23
+ export async function specHash(l) {
24
+ const files = [...(await filesUnder(l.referee)).filter((f) => f !== approvalFile(l)), l.factsFile].sort();
25
+ const hash = createHash('sha256');
26
+ for (const file of files) {
27
+ hash.update(path.relative(l.root, file));
28
+ hash.update('\0');
29
+ hash.update(await readFile(file));
30
+ hash.update('\0');
31
+ }
32
+ return hash.digest('hex');
33
+ }
34
+ export async function approve(l, options) {
35
+ if (!(options.budgetUsd > 0))
36
+ throw new Error('An approval needs a dollar budget for the simulations.');
37
+ await readFile(l.specFile).catch(() => { throw new Error('There is no derived spec in this project to approve.'); });
38
+ const approval = {
39
+ schema: 'sdd-approval/1',
40
+ hash: await specHash(l),
41
+ approvedAt: new Date().toISOString(),
42
+ budgetUsd: options.budgetUsd,
43
+ note: options.note?.trim() || undefined,
44
+ };
45
+ await writeFile(approvalFile(l), JSON.stringify(approval, null, 2) + '\n');
46
+ return approval;
47
+ }
48
+ /** Whether the spec as it stands now is the one the owner approved. */
49
+ export async function approvalState(l) {
50
+ let approval;
51
+ try {
52
+ approval = JSON.parse(await readFile(approvalFile(l), 'utf8'));
53
+ }
54
+ catch {
55
+ return { approved: false, reason: 'The owner has not approved the spec.' };
56
+ }
57
+ if (approval.hash !== (await specHash(l))) {
58
+ return { approved: false, reason: 'The spec, the personas or the ledger changed after the owner approved them. It needs approving again.' };
59
+ }
60
+ return { approved: true, approval };
61
+ }
62
+ /** Throws unless the current spec is approved. Every later step calls this first. */
63
+ export async function requireApproval(l) {
64
+ const state = await approvalState(l);
65
+ if (!state.approved)
66
+ throw new Error(state.reason);
67
+ return state.approval;
68
+ }