sdd-engine 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +165 -2
- package/dist/engine/cli.js +95 -0
- package/dist/engine/simulate.js +214 -0
- package/dist/judge/conversation.js +45 -0
- package/dist/judge/judge.js +122 -0
- package/dist/judge/outcome.js +42 -0
- package/dist/judge/regrade.js +74 -0
- package/dist/pipeline/agent.js +78 -0
- package/dist/pipeline/agentstage.js +88 -0
- package/dist/pipeline/approve.js +68 -0
- package/dist/pipeline/cli.js +261 -0
- package/dist/pipeline/derive.js +287 -0
- package/dist/pipeline/designloop.js +385 -0
- package/dist/pipeline/facts.js +108 -0
- package/dist/pipeline/mock.js +253 -0
- package/dist/pipeline/project.js +79 -0
- package/dist/pipeline/render.js +95 -0
- package/dist/pipeline/spec.js +272 -0
- package/dist/pipeline/try.js +162 -0
- package/dist/runner/batch-lib.js +157 -0
- package/dist/runner/batch.js +56 -0
- package/dist/runner/cli.js +61 -0
- package/dist/runner/llm/claude-code.js +123 -0
- package/dist/runner/llm/index.js +12 -0
- package/dist/runner/llm/types.js +17 -0
- package/dist/runner/reader/answer.js +76 -0
- package/dist/runner/reader/extract.js +106 -0
- package/dist/runner/reader/run.js +111 -0
- package/dist/runner/reader/skim.js +115 -0
- package/dist/runner/rng.js +22 -0
- package/dist/runner/sandbox/cli.js +60 -0
- package/dist/runner/sandbox/sandbox.js +144 -0
- package/dist/runner/sandbox/server.js +64 -0
- package/dist/runner/spec.js +84 -0
- package/dist/runner/talker/decide.js +110 -0
- package/dist/runner/talker/run.js +159 -0
- package/dist/runner/talker/world.js +36 -0
- package/dist/runner/trace.js +130 -0
- package/dist/runner/visitor/decide.js +159 -0
- package/dist/runner/visitor/observe.js +102 -0
- package/dist/runner/visitor/run.js +296 -0
- package/dist/scorer/compare.js +119 -0
- package/dist/scorer/score.js +201 -0
- package/dist/scorer/spot-check-compare.js +41 -0
- package/dist/scorer/spot-check-view.js +45 -0
- package/dist/scorer/stats.js +45 -0
- package/dist/scorer/summary.js +62 -0
- package/package.json +49 -4
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// Prints a spot-check sheet without the judge's grades, for blind regrading.
|
|
2
|
+
// node scorer/spot-check-view.ts <sheet.csv> <question id>
|
|
3
|
+
import { readFileSync } from 'node:fs';
|
|
4
|
+
export function parseCsv(text) {
|
|
5
|
+
const rows = [];
|
|
6
|
+
let row = [], cell = '', quoted = false;
|
|
7
|
+
for (let i = 0; i < text.length; i++) {
|
|
8
|
+
const c = text[i];
|
|
9
|
+
if (quoted) {
|
|
10
|
+
if (c === '"' && text[i + 1] === '"') {
|
|
11
|
+
cell += '"';
|
|
12
|
+
i++;
|
|
13
|
+
}
|
|
14
|
+
else if (c === '"')
|
|
15
|
+
quoted = false;
|
|
16
|
+
else
|
|
17
|
+
cell += c;
|
|
18
|
+
}
|
|
19
|
+
else if (c === '"')
|
|
20
|
+
quoted = true;
|
|
21
|
+
else if (c === ',') {
|
|
22
|
+
row.push(cell);
|
|
23
|
+
cell = '';
|
|
24
|
+
}
|
|
25
|
+
else if (c === '\n') {
|
|
26
|
+
row.push(cell);
|
|
27
|
+
rows.push(row);
|
|
28
|
+
row = [];
|
|
29
|
+
cell = '';
|
|
30
|
+
}
|
|
31
|
+
else
|
|
32
|
+
cell += c;
|
|
33
|
+
}
|
|
34
|
+
if (cell || row.length) {
|
|
35
|
+
row.push(cell);
|
|
36
|
+
rows.push(row);
|
|
37
|
+
}
|
|
38
|
+
return rows;
|
|
39
|
+
}
|
|
40
|
+
if (import.meta.main) {
|
|
41
|
+
const [file, question] = process.argv.slice(2);
|
|
42
|
+
const rows = parseCsv(readFileSync(file, 'utf8')).slice(1);
|
|
43
|
+
rows.forEach((r, i) => { if (r[1] === question)
|
|
44
|
+
console.log(`${i + 1}. ${r[3]}`); });
|
|
45
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// Statistics for the scorer.
|
|
2
|
+
/** Wilson score interval for a proportion. z = 1.96 gives 95%. */
|
|
3
|
+
export function wilson(passed, n, z = 1.96) {
|
|
4
|
+
if (n === 0)
|
|
5
|
+
return { passed, n, rate: NaN, lower: 0, upper: 1 };
|
|
6
|
+
const p = passed / n;
|
|
7
|
+
const z2 = z * z;
|
|
8
|
+
const denom = 1 + z2 / n;
|
|
9
|
+
const centre = (p + z2 / (2 * n)) / denom;
|
|
10
|
+
const half = (z * Math.sqrt((p * (1 - p)) / n + z2 / (4 * n * n))) / denom;
|
|
11
|
+
return { passed, n, rate: p, lower: Math.max(0, centre - half), upper: Math.min(1, centre + half) };
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* A threshold is met only when the lower bound clears it. When the interval
|
|
15
|
+
* straddles the threshold the honest answer is that the data cannot say.
|
|
16
|
+
*/
|
|
17
|
+
export function verdict(interval, threshold) {
|
|
18
|
+
if (interval.lower >= threshold)
|
|
19
|
+
return 'met';
|
|
20
|
+
if (interval.upper < threshold)
|
|
21
|
+
return 'not met';
|
|
22
|
+
return 'inside the noise';
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Paired comparison of two binary outcomes over n pairs.
|
|
26
|
+
* gained: pairs where A failed and B passed. lost: the reverse.
|
|
27
|
+
*/
|
|
28
|
+
export function pairedDifference(gained, lost, n, z = 1.96) {
|
|
29
|
+
const difference = (gained - lost) / n;
|
|
30
|
+
const se = Math.sqrt(gained + lost - (gained - lost) ** 2 / n) / n;
|
|
31
|
+
// Exact binomial test: under no difference, each flip is a coin toss.
|
|
32
|
+
const flips = gained + lost;
|
|
33
|
+
let p = 1;
|
|
34
|
+
if (flips > 0) {
|
|
35
|
+
const k = Math.min(gained, lost);
|
|
36
|
+
let tail = 0;
|
|
37
|
+
let term = 0.5 ** flips; // C(flips, 0) / 2^flips
|
|
38
|
+
for (let i = 0; i <= k; i++) {
|
|
39
|
+
tail += term;
|
|
40
|
+
term = (term * (flips - i)) / (i + 1);
|
|
41
|
+
}
|
|
42
|
+
p = Math.min(1, 2 * tail);
|
|
43
|
+
}
|
|
44
|
+
return { difference, lower: difference - z * se, upper: difference + z * se, p };
|
|
45
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// A compact, structured summary of one persona set's results on one page.
|
|
2
|
+
// Used by the automated loop to decide, and to brief the revising agent.
|
|
3
|
+
import { createRng, deriveSeed } from "../runner/rng.js";
|
|
4
|
+
import { verdict, wilson } from "./stats.js";
|
|
5
|
+
export function summarize(spec, thresholds, traces, set, maxSeed = Infinity) {
|
|
6
|
+
const runs = traces.filter((t) => t.personaSet === set && t.seed <= maxSeed);
|
|
7
|
+
const passed = (t, id) => t.grades.find((g) => g.questionId === id).pass;
|
|
8
|
+
const rate = (group, id) => (group.length ? group.filter((t) => passed(t, id)).length / group.length : NaN);
|
|
9
|
+
const questions = spec.questions.map((q) => {
|
|
10
|
+
const interval = wilson(runs.filter((t) => passed(t, q.id)).length, runs.length);
|
|
11
|
+
return { id: q.id, threshold: thresholds[q.id], interval, verdict: verdict(interval, thresholds[q.id]) };
|
|
12
|
+
});
|
|
13
|
+
const group = (f) => {
|
|
14
|
+
const out = {};
|
|
15
|
+
for (const name of new Set(runs.map(f))) {
|
|
16
|
+
const members = runs.filter((t) => f(t) === name);
|
|
17
|
+
out[name] = Object.fromEntries(spec.questions.map((q) => [q.id, round(rate(members, q.id))]));
|
|
18
|
+
}
|
|
19
|
+
return out;
|
|
20
|
+
};
|
|
21
|
+
const stoppedIn = {};
|
|
22
|
+
for (const t of runs)
|
|
23
|
+
stoppedIn[t.deepestSection] = (stoppedIn[t.deepestSection] ?? 0) + 1 / runs.length;
|
|
24
|
+
for (const k of Object.keys(stoppedIn))
|
|
25
|
+
stoppedIn[k] = round(stoppedIn[k]);
|
|
26
|
+
const rng = createRng(deriveSeed('failing-sample', spec.experiment, runs[0]?.page.sha256 ?? ''));
|
|
27
|
+
const failingAnswers = spec.questions.flatMap((q) => {
|
|
28
|
+
const failures = runs.filter((t) => !passed(t, q.id));
|
|
29
|
+
return failures
|
|
30
|
+
.map((t) => ({ t, r: rng() }))
|
|
31
|
+
.sort((a, b) => a.r - b.r)
|
|
32
|
+
.slice(0, 6)
|
|
33
|
+
.map(({ t }) => ({
|
|
34
|
+
question: q.id,
|
|
35
|
+
answer: t.answers.find((a) => a.id === q.id).answer,
|
|
36
|
+
judgeNote: t.grades.find((g) => g.questionId === q.id).note,
|
|
37
|
+
}));
|
|
38
|
+
});
|
|
39
|
+
return {
|
|
40
|
+
set,
|
|
41
|
+
runs: runs.length,
|
|
42
|
+
questions,
|
|
43
|
+
shortfall: round(questions.reduce((sum, q) => sum + Math.max(0, q.threshold - q.interval.rate), 0)),
|
|
44
|
+
minMargin: round(Math.min(...questions.map((q) => q.interval.rate - q.threshold))),
|
|
45
|
+
met: runs.length > 0 && questions.every((q) => q.verdict === 'met'),
|
|
46
|
+
ratesClear: runs.length > 0 && questions.every((q) => q.interval.rate >= q.threshold),
|
|
47
|
+
byViewport: group((t) => t.viewport.name),
|
|
48
|
+
byPersona: group((t) => t.persona.id),
|
|
49
|
+
missedElements: spec.questions
|
|
50
|
+
.flatMap((q) => q.rubric.required.map((element, i) => ({
|
|
51
|
+
question: q.id,
|
|
52
|
+
element,
|
|
53
|
+
missedRate: round(runs.filter((t) => !t.grades.find((g) => g.questionId === q.id).requiredMet[i]).length / runs.length),
|
|
54
|
+
})))
|
|
55
|
+
.sort((a, b) => b.missedRate - a.missedRate),
|
|
56
|
+
stoppedIn,
|
|
57
|
+
failingAnswers,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
function round(n) {
|
|
61
|
+
return Math.round(n * 1000) / 1000;
|
|
62
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,51 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sdd-engine",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Simulation Driven Development engine: simulated visitors use a local app and score it against thresholds",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"author": "Chad Cox",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/TurkeyCode-ai/sdd.git"
|
|
11
|
+
},
|
|
12
|
+
"bin": {
|
|
13
|
+
"sdd": "dist/engine/cli.js"
|
|
14
|
+
},
|
|
15
|
+
"files": [
|
|
16
|
+
"dist",
|
|
17
|
+
"README.md",
|
|
18
|
+
"LICENSE"
|
|
19
|
+
],
|
|
20
|
+
"engines": {
|
|
21
|
+
"node": ">=24"
|
|
22
|
+
},
|
|
23
|
+
"scripts": {
|
|
24
|
+
"prebuild": "node -e \"import('node:fs').then(fs => fs.rmSync('dist', { recursive: true, force: true }))\"",
|
|
25
|
+
"build": "tsc -p tsconfig.build.json",
|
|
26
|
+
"postbuild": "node -e \"import('node:fs').then(fs => fs.chmodSync('dist/engine/cli.js', 0o755))\"",
|
|
27
|
+
"prepublishOnly": "npm run typecheck && npm test && npm run build",
|
|
28
|
+
"typecheck": "tsc --noEmit",
|
|
29
|
+
"test": "node --test \"engine/**/*.test.ts\" \"pipeline/**/*.test.ts\" \"runner/**/*.test.ts\" \"judge/**/*.test.ts\" \"scorer/**/*.test.ts\" \"loop/**/*.test.ts\"",
|
|
30
|
+
"sdd": "node engine/cli.ts",
|
|
31
|
+
"simulate": "node engine/cli.ts simulate",
|
|
32
|
+
"example:waitlist": "node examples/waitlist/app.ts",
|
|
33
|
+
"sandbox": "node runner/sandbox/cli.ts serve",
|
|
34
|
+
"sandbox:check": "node runner/sandbox/cli.ts check",
|
|
35
|
+
"vendor-fonts": "node scripts/vendor-fonts.ts",
|
|
36
|
+
"reader": "node runner/cli.ts reader",
|
|
37
|
+
"batch": "node runner/batch.ts",
|
|
38
|
+
"score": "node scorer/score.ts",
|
|
39
|
+
"regrade": "node judge/regrade.ts",
|
|
40
|
+
"compare": "node scorer/compare.ts",
|
|
41
|
+
"loop": "node loop/loop.ts"
|
|
42
|
+
},
|
|
43
|
+
"dependencies": {
|
|
44
|
+
"playwright": "^1.56.0",
|
|
45
|
+
"yaml": "^2.9.1"
|
|
46
|
+
},
|
|
47
|
+
"devDependencies": {
|
|
48
|
+
"@types/node": "^24.0.0",
|
|
49
|
+
"typescript": "^5.9.0"
|
|
50
|
+
}
|
|
51
|
+
}
|