sdd-engine 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +178 -2
- package/dist/engine/cli.js +95 -0
- package/dist/engine/simulate.js +214 -0
- package/dist/judge/conversation.js +45 -0
- package/dist/judge/judge.js +122 -0
- package/dist/judge/outcome.js +42 -0
- package/dist/judge/regrade.js +74 -0
- package/dist/pipeline/agent.js +78 -0
- package/dist/pipeline/agentstage.js +88 -0
- package/dist/pipeline/approve.js +68 -0
- package/dist/pipeline/cli.js +261 -0
- package/dist/pipeline/derive.js +287 -0
- package/dist/pipeline/designloop.js +385 -0
- package/dist/pipeline/facts.js +108 -0
- package/dist/pipeline/mock.js +253 -0
- package/dist/pipeline/project.js +79 -0
- package/dist/pipeline/render.js +95 -0
- package/dist/pipeline/spec.js +272 -0
- package/dist/pipeline/try.js +162 -0
- package/dist/runner/batch-lib.js +157 -0
- package/dist/runner/batch.js +56 -0
- package/dist/runner/cli.js +61 -0
- package/dist/runner/llm/claude-code.js +123 -0
- package/dist/runner/llm/index.js +12 -0
- package/dist/runner/llm/types.js +17 -0
- package/dist/runner/reader/answer.js +76 -0
- package/dist/runner/reader/extract.js +106 -0
- package/dist/runner/reader/run.js +111 -0
- package/dist/runner/reader/skim.js +115 -0
- package/dist/runner/rng.js +22 -0
- package/dist/runner/sandbox/cli.js +60 -0
- package/dist/runner/sandbox/sandbox.js +144 -0
- package/dist/runner/sandbox/server.js +64 -0
- package/dist/runner/spec.js +84 -0
- package/dist/runner/talker/decide.js +110 -0
- package/dist/runner/talker/run.js +159 -0
- package/dist/runner/talker/world.js +36 -0
- package/dist/runner/trace.js +130 -0
- package/dist/runner/visitor/decide.js +159 -0
- package/dist/runner/visitor/observe.js +102 -0
- package/dist/runner/visitor/run.js +296 -0
- package/dist/scorer/compare.js +119 -0
- package/dist/scorer/score.js +201 -0
- package/dist/scorer/spot-check-compare.js +41 -0
- package/dist/scorer/spot-check-view.js +45 -0
- package/dist/scorer/stats.js +45 -0
- package/dist/scorer/summary.js +62 -0
- package/package.json +49 -4
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
// Visitor mode: one persona, one session, one trace.
|
|
2
|
+
//
|
|
3
|
+
// The visitor lands on the entry page and acts: scroll, click, type, or
|
|
4
|
+
// leave. Leaving is the default (see decide.ts). The engine, not the model,
|
|
5
|
+
// decides whether the task was completed, from what the browser actually did.
|
|
6
|
+
//
|
|
7
|
+
// Use this only against a copy of the app that is safe to act on. Visitors
|
|
8
|
+
// submit forms. The sandbox still blocks every request to any other origin.
|
|
9
|
+
import { createHash } from 'node:crypto';
|
|
10
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
11
|
+
import path from 'node:path';
|
|
12
|
+
import { judgeOutcome } from "../../judge/outcome.js";
|
|
13
|
+
import { captureScreens, extractLayout } from "../reader/extract.js";
|
|
14
|
+
import { VIEWPORTS } from "../reader/run.js";
|
|
15
|
+
import { countWords, splitSentences, UNCALIBRATED_DIALS } from "../reader/skim.js";
|
|
16
|
+
import { createRng, deriveSeed } from "../rng.js";
|
|
17
|
+
import { validateTrace } from "../trace.js";
|
|
18
|
+
import { decide } from "./decide.js";
|
|
19
|
+
import { observe } from "./observe.js";
|
|
20
|
+
const CLICK_SECONDS = 1.2;
|
|
21
|
+
const LOAD_SECONDS = 1;
|
|
22
|
+
const TYPE_SECONDS_PER_CHAR = 0.25;
|
|
23
|
+
/**
|
|
24
|
+
* Chance of missing a tap. Small targets are missed more often, in the
|
|
25
|
+
* spirit of Fitts's law: 44 CSS pixels is a comfortable target, and the miss
|
|
26
|
+
* rate grows as the target shrinks below that.
|
|
27
|
+
*/
|
|
28
|
+
export function missChance(element, baseRate) {
|
|
29
|
+
const size = Math.max(1, Math.min(element.width, element.height));
|
|
30
|
+
return Math.min(0.5, baseRate * Math.max(1, 44 / size));
|
|
31
|
+
}
|
|
32
|
+
/** Headings are always taken in. Each body sentence is taken in with the skim probability, once per session. */
|
|
33
|
+
function skimScreen(o, pRead, rng, memo) {
|
|
34
|
+
const seen = [];
|
|
35
|
+
let words = 0;
|
|
36
|
+
for (const block of o.text) {
|
|
37
|
+
const parts = block.kind === 'heading' ? [block.text] : splitSentences(block.text);
|
|
38
|
+
for (const text of parts) {
|
|
39
|
+
const key = `${o.url}|${text}`;
|
|
40
|
+
const first = !memo.has(key);
|
|
41
|
+
if (first)
|
|
42
|
+
memo.set(key, block.kind === 'heading' || rng() < pRead);
|
|
43
|
+
if (!memo.get(key))
|
|
44
|
+
continue;
|
|
45
|
+
seen.push({ kind: block.kind, text });
|
|
46
|
+
if (first)
|
|
47
|
+
words += countWords(text); // reading time is paid once
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return { seen, words };
|
|
51
|
+
}
|
|
52
|
+
export async function runVisitor(options) {
|
|
53
|
+
const { spec, persona, sandbox, llm, seed } = options;
|
|
54
|
+
if (!spec.task || !spec.visitor)
|
|
55
|
+
throw new Error('Visitor mode needs a task spec');
|
|
56
|
+
const viewportOptions = VIEWPORTS[options.viewport];
|
|
57
|
+
if (!viewportOptions)
|
|
58
|
+
throw new Error(`Unknown viewport "${options.viewport}"`);
|
|
59
|
+
const dials = { ...(options.dials ?? UNCALIBRATED_DIALS), timeBudgetSeconds: spec.visitor.patience_seconds };
|
|
60
|
+
const derivedSeed = deriveSeed(spec.experiment, persona.id, options.viewport, seed);
|
|
61
|
+
const rng = createRng(derivedSeed);
|
|
62
|
+
const startedAt = new Date().toISOString();
|
|
63
|
+
const budget = spec.visitor.patience_seconds;
|
|
64
|
+
const maxSteps = spec.visitor.max_steps ?? 25;
|
|
65
|
+
const pRead = 1 - Math.min(1, Math.max(0, persona.dials.skim_rate * dials.skimScale));
|
|
66
|
+
const missRate = persona.dials.mis_tap_rate ?? 0.03;
|
|
67
|
+
const entry = options.entryPath ?? '/';
|
|
68
|
+
const success = spec.task.success;
|
|
69
|
+
const steps = [];
|
|
70
|
+
const history = [];
|
|
71
|
+
const calls = [];
|
|
72
|
+
const memo = new Map();
|
|
73
|
+
let t = 0;
|
|
74
|
+
let outcome;
|
|
75
|
+
let outcomeNote = '';
|
|
76
|
+
let requestSeen = false;
|
|
77
|
+
let pressedWanted = '';
|
|
78
|
+
let believedDone = false;
|
|
79
|
+
let finalScreen = { url: '', title: '', text: [] };
|
|
80
|
+
let pageHash = '';
|
|
81
|
+
let pageWords = 0;
|
|
82
|
+
const context = await sandbox.newContext(viewportOptions);
|
|
83
|
+
try {
|
|
84
|
+
if (success.request) {
|
|
85
|
+
const want = success.request;
|
|
86
|
+
context.on('request', (request) => {
|
|
87
|
+
const url = new URL(request.url());
|
|
88
|
+
if (url.origin === sandbox.origin && url.pathname.includes(want.path_includes) && (!want.method || request.method() === want.method.toUpperCase())) {
|
|
89
|
+
requestSeen = true;
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
const page = await context.newPage();
|
|
94
|
+
const response = await page.goto(`${sandbox.origin}${entry}`, { waitUntil: 'load' });
|
|
95
|
+
if (response?.status() !== 200)
|
|
96
|
+
throw new Error(`Entry page returned HTTP ${response?.status()}`);
|
|
97
|
+
pageHash = createHash('sha256').update(await response.body()).digest('hex');
|
|
98
|
+
t += LOAD_SECONDS;
|
|
99
|
+
const completed = async (p) => {
|
|
100
|
+
if (success.request && requestSeen)
|
|
101
|
+
return true;
|
|
102
|
+
if (pressedWanted)
|
|
103
|
+
return true;
|
|
104
|
+
if (success.url_includes && new URL(p.url()).pathname.includes(success.url_includes))
|
|
105
|
+
return true;
|
|
106
|
+
if (success.text_visible && (await p.getByText(success.text_visible, { exact: false }).first().isVisible().catch(() => false)))
|
|
107
|
+
return true;
|
|
108
|
+
return false;
|
|
109
|
+
};
|
|
110
|
+
const settle = async () => {
|
|
111
|
+
await page.waitForLoadState('load', { timeout: 5000 }).catch(() => { });
|
|
112
|
+
await page.waitForLoadState('networkidle', { timeout: 3000 }).catch(() => { });
|
|
113
|
+
};
|
|
114
|
+
while (!outcome) {
|
|
115
|
+
const start = t;
|
|
116
|
+
const o = await observe(page);
|
|
117
|
+
finalScreen = { url: o.url, title: o.title, text: o.text.map((b) => b.text) };
|
|
118
|
+
if (steps.length === 0)
|
|
119
|
+
pageWords = countWords(o.text.map((b) => b.text).join(' '));
|
|
120
|
+
const { seen, words } = skimScreen(o, pRead, rng, memo);
|
|
121
|
+
t += (words / persona.dials.reading_wpm) * 60;
|
|
122
|
+
const record = (action, cites, reason, result, extra = {}) => {
|
|
123
|
+
steps.push({
|
|
124
|
+
index: steps.length, url: o.url, scrollY: o.scrollY,
|
|
125
|
+
visibleText: o.text, seenText: seen,
|
|
126
|
+
elements: o.elements.map(({ n, kind, label }) => ({ n, kind, label })),
|
|
127
|
+
action, cites, reason, result,
|
|
128
|
+
simulatedSecondsStart: round(start), simulatedSecondsEnd: round(Math.min(t, budget)),
|
|
129
|
+
...extra,
|
|
130
|
+
});
|
|
131
|
+
};
|
|
132
|
+
if (t >= budget) {
|
|
133
|
+
t = budget;
|
|
134
|
+
record('stop', '', 'patience ran out', 'time up');
|
|
135
|
+
outcome = 'time_up';
|
|
136
|
+
outcomeNote = 'ran out of patience';
|
|
137
|
+
break;
|
|
138
|
+
}
|
|
139
|
+
if (steps.length >= maxSteps) {
|
|
140
|
+
record('stop', '', 'step limit reached', 'gave up');
|
|
141
|
+
outcome = 'gave_up';
|
|
142
|
+
outcomeNote = `still going after ${maxSteps} steps`;
|
|
143
|
+
break;
|
|
144
|
+
}
|
|
145
|
+
let screenshot;
|
|
146
|
+
if (spec.visitor.screenshots) {
|
|
147
|
+
await extractLayout(page); // marks body text so the screenshot can blur it
|
|
148
|
+
const [screen] = await captureScreens(page, [o.scrollY], viewportOptions.viewport.width > 600 ? 640 : 300);
|
|
149
|
+
const bytes = Buffer.from(screen.base64, 'base64');
|
|
150
|
+
const sha = createHash('sha256').update(bytes).digest('hex');
|
|
151
|
+
const ref = path.join('shots', spec.experiment, 'visitor', `${sha.slice(0, 16)}.jpg`);
|
|
152
|
+
await mkdir(path.dirname(path.resolve(options.repoRoot, ref)), { recursive: true });
|
|
153
|
+
await writeFile(path.resolve(options.repoRoot, ref), bytes, { flag: 'wx' }).catch((err) => {
|
|
154
|
+
if (err.code !== 'EEXIST')
|
|
155
|
+
throw err;
|
|
156
|
+
});
|
|
157
|
+
screenshot = { mediaType: screen.mediaType, base64: screen.base64, ref, sha256: sha };
|
|
158
|
+
}
|
|
159
|
+
const decision = await decide(llm, {
|
|
160
|
+
persona, instruction: spec.task.instruction, observation: o, seenText: seen, history, secondsLeft: budget - t, screenshot,
|
|
161
|
+
});
|
|
162
|
+
calls.push(...decision.calls);
|
|
163
|
+
const a = decision.action;
|
|
164
|
+
const element = 'element' in a ? o.elements.find((e) => e.n === a.element) : undefined;
|
|
165
|
+
const describe = element ? `${a.action} ${element.kind} "${element.label}"` : a.action;
|
|
166
|
+
if (a.action === 'done') {
|
|
167
|
+
// They think they finished. Whether they did is checked below, not taken on trust.
|
|
168
|
+
record('done', a.cites, a.reason, 'stopped, believing the task was done');
|
|
169
|
+
believedDone = true;
|
|
170
|
+
outcome = (await completed(page)) ? 'completed' : 'thought_done';
|
|
171
|
+
outcomeNote = outcome === 'completed' ? 'the task was completed' : 'believed the task was done, but the success check did not pass';
|
|
172
|
+
break;
|
|
173
|
+
}
|
|
174
|
+
if (a.action === 'leave') {
|
|
175
|
+
record('leave', a.cites, a.reason, 'left');
|
|
176
|
+
outcome = 'left';
|
|
177
|
+
outcomeNote = a.reason;
|
|
178
|
+
break;
|
|
179
|
+
}
|
|
180
|
+
let result;
|
|
181
|
+
if (a.action === 'scroll') {
|
|
182
|
+
const before = o.scrollY;
|
|
183
|
+
await page.evaluate((dy) => window.scrollBy(0, dy), Math.round(o.viewportHeight * dials.scrollFraction));
|
|
184
|
+
t += dials.scrollSeconds;
|
|
185
|
+
const after = await page.evaluate(() => Math.round(window.scrollY));
|
|
186
|
+
result = after > before ? 'scrolled down' : 'nothing more below';
|
|
187
|
+
}
|
|
188
|
+
else if (element.kind === 'link' && element.goesTo === 'outside') {
|
|
189
|
+
// Leaving for another site ends the visit. The request is never made.
|
|
190
|
+
record(a.action, a.cites, a.reason, 'left for another site', { element: element.n });
|
|
191
|
+
// Some pages exist to send people somewhere. Then leaving for that place is the goal.
|
|
192
|
+
const wanted = (success.leaves_to ?? []).find((part) => (element.href ?? '').toLowerCase().includes(part.toLowerCase()));
|
|
193
|
+
if (wanted) {
|
|
194
|
+
outcome = 'completed';
|
|
195
|
+
outcomeNote = `followed the link the page is for: "${element.label}"`;
|
|
196
|
+
}
|
|
197
|
+
else {
|
|
198
|
+
outcome = 'left';
|
|
199
|
+
outcomeNote = `followed a link off the site: "${element.label}"`;
|
|
200
|
+
}
|
|
201
|
+
break;
|
|
202
|
+
}
|
|
203
|
+
else if (rng() < missChance(element, missRate)) {
|
|
204
|
+
t += CLICK_SECONDS;
|
|
205
|
+
result = 'missed the tap, nothing happened';
|
|
206
|
+
}
|
|
207
|
+
else if (a.action === 'type') {
|
|
208
|
+
await page.locator(`[data-sdd-el="${element.n}"]`).fill(a.text);
|
|
209
|
+
t += 1 + a.text.length * TYPE_SECONDS_PER_CHAR;
|
|
210
|
+
result = 'typed it in';
|
|
211
|
+
}
|
|
212
|
+
else {
|
|
213
|
+
const urlBefore = page.url();
|
|
214
|
+
await page.locator(`[data-sdd-el="${element.n}"]`).click({ timeout: 5000 }).catch(() => { });
|
|
215
|
+
await settle();
|
|
216
|
+
t += CLICK_SECONDS;
|
|
217
|
+
const moved = page.url() !== urlBefore;
|
|
218
|
+
if (moved)
|
|
219
|
+
t += LOAD_SECONDS;
|
|
220
|
+
result = moved ? `went to ${new URL(page.url()).pathname}` : 'clicked, stayed on the same page';
|
|
221
|
+
const label = element.label.toLowerCase();
|
|
222
|
+
if ((success.clicks ?? []).some((part) => label.includes(part.toLowerCase())))
|
|
223
|
+
pressedWanted = element.label;
|
|
224
|
+
}
|
|
225
|
+
record(a.action, a.cites, a.reason, result, { element: element?.n, typed: a.action === 'type' ? a.text : undefined });
|
|
226
|
+
history.push({ action: describe + (a.action === 'type' ? ` with "${a.text}"` : ''), result });
|
|
227
|
+
sandbox.assertClean(); // hard rule 1: an outside request fails the run
|
|
228
|
+
if (await completed(page)) {
|
|
229
|
+
outcome = 'completed';
|
|
230
|
+
outcomeNote = 'the task was completed';
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
finally {
|
|
235
|
+
await context.close();
|
|
236
|
+
}
|
|
237
|
+
sandbox.assertClean();
|
|
238
|
+
// A prose outcome is judged from the final screen, however the session
|
|
239
|
+
// ended. The judge sees the screen and the expected outcome, nothing else.
|
|
240
|
+
if (success.outcome && outcome !== 'completed') {
|
|
241
|
+
const verdict = await judgeOutcome(llm, success.outcome, finalScreen);
|
|
242
|
+
calls.push(...verdict.calls);
|
|
243
|
+
if (verdict.met) {
|
|
244
|
+
outcome = 'completed';
|
|
245
|
+
outcomeNote = `the final screen shows the outcome: ${verdict.note}`;
|
|
246
|
+
}
|
|
247
|
+
else if (believedDone) {
|
|
248
|
+
outcome = 'thought_done';
|
|
249
|
+
outcomeNote = `believed the task was done, but the final screen does not show it: ${verdict.note}`;
|
|
250
|
+
}
|
|
251
|
+
else {
|
|
252
|
+
outcomeNote = `${outcomeNote}. Final screen: ${verdict.note}`;
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
const size = viewportOptions.viewport;
|
|
256
|
+
const usd = calls.reduce((sum, c) => sum + c.response.costUsd, 0);
|
|
257
|
+
const done = outcome === 'completed';
|
|
258
|
+
const last = steps.at(-1);
|
|
259
|
+
const trace = {
|
|
260
|
+
schema: 'sdd-trace/1',
|
|
261
|
+
runId: `${startedAt.replace(/[:.]/g, '-')}_${persona.id}_${options.viewport}_s${seed}`,
|
|
262
|
+
...(options.batch ? { batch: options.batch } : {}),
|
|
263
|
+
experiment: spec.experiment,
|
|
264
|
+
mode: 'visitor',
|
|
265
|
+
startedAt,
|
|
266
|
+
page: { path: `${sandbox.origin}${entry}`, sha256: pageHash, words: pageWords },
|
|
267
|
+
persona,
|
|
268
|
+
personaSet: options.personaSet ?? 'working',
|
|
269
|
+
viewport: { name: options.viewport, width: size.width, height: size.height },
|
|
270
|
+
seed,
|
|
271
|
+
derivedSeed,
|
|
272
|
+
dials: { ...dials, calibrated: false },
|
|
273
|
+
steps: [],
|
|
274
|
+
visitorSteps: steps,
|
|
275
|
+
recalled: [],
|
|
276
|
+
outcome: outcome,
|
|
277
|
+
simulatedSeconds: round(Math.min(t, budget)),
|
|
278
|
+
// "Where they stopped": the page they were on when the visit ended.
|
|
279
|
+
deepestSection: last ? last.url : entry,
|
|
280
|
+
answers: [{ id: 'task', ask: spec.task.goal, answer: `${outcome}: ${outcomeNote}` }],
|
|
281
|
+
// Graded by the engine from what the browser did. No judge model is involved.
|
|
282
|
+
grades: [{ questionId: 'task', pass: done, requiredMet: [done], bonusMet: [], note: done ? 'complete' : outcomeNote }],
|
|
283
|
+
llmCalls: calls,
|
|
284
|
+
cost: {
|
|
285
|
+
usd,
|
|
286
|
+
inputTokens: calls.reduce((s, c) => s + c.response.usage.inputTokens + c.response.usage.cacheReadTokens + c.response.usage.cacheWriteTokens, 0),
|
|
287
|
+
outputTokens: calls.reduce((s, c) => s + c.response.usage.outputTokens, 0),
|
|
288
|
+
},
|
|
289
|
+
sandbox: sandbox.record(),
|
|
290
|
+
};
|
|
291
|
+
validateTrace(trace, ['task']);
|
|
292
|
+
return trace;
|
|
293
|
+
}
|
|
294
|
+
function round(n) {
|
|
295
|
+
return Math.round(n * 1000) / 1000;
|
|
296
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
// Paired comparison of two page variants.
|
|
2
|
+
//
|
|
3
|
+
// node scorer/compare.ts --a <batch>[:<grades label>] --b <batch>[:<grades label>]
|
|
4
|
+
//
|
|
5
|
+
// A is the baseline, B the candidate. Runs are paired on persona, viewport,
|
|
6
|
+
// and seed. Writes reports/<experiment>/compare-<a>-vs-<b>.md.
|
|
7
|
+
import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
|
|
8
|
+
import path from 'node:path';
|
|
9
|
+
import { parseArgs } from 'node:util';
|
|
10
|
+
import { parse } from 'yaml';
|
|
11
|
+
import { loadSpec } from "../runner/spec.js";
|
|
12
|
+
import { pairedDifference, verdict, wilson } from "./stats.js";
|
|
13
|
+
const repoRoot = path.resolve(import.meta.dirname, '..');
|
|
14
|
+
const { values } = parseArgs({
|
|
15
|
+
options: { spec: { type: 'string', default: 'specs/sdd-site.spec.yaml' }, a: { type: 'string' }, b: { type: 'string' } },
|
|
16
|
+
});
|
|
17
|
+
if (!values.a || !values.b) {
|
|
18
|
+
console.error('Usage: node scorer/compare.ts --a <batch>[:<grades label>] --b <batch>[:<grades label>]');
|
|
19
|
+
process.exit(2);
|
|
20
|
+
}
|
|
21
|
+
const specFile = path.resolve(repoRoot, values.spec);
|
|
22
|
+
const spec = await loadSpec(specFile);
|
|
23
|
+
const pass = parse(await readFile(specFile, 'utf8')).pass;
|
|
24
|
+
async function loadBatch(spec, ref) {
|
|
25
|
+
const [batch, label] = ref.split(':');
|
|
26
|
+
const dir = path.join(repoRoot, 'traces', spec.experiment);
|
|
27
|
+
const traces = [];
|
|
28
|
+
for (const file of (await readdir(dir)).sort()) {
|
|
29
|
+
if (!file.endsWith('.json'))
|
|
30
|
+
continue;
|
|
31
|
+
const trace = JSON.parse(await readFile(path.join(dir, file), 'utf8'));
|
|
32
|
+
if (trace.batch !== batch)
|
|
33
|
+
continue;
|
|
34
|
+
if (label) {
|
|
35
|
+
const regrade = JSON.parse(await readFile(path.join(repoRoot, 'grades', spec.experiment, batch, label, file), 'utf8'));
|
|
36
|
+
trace.grades = regrade.grades;
|
|
37
|
+
}
|
|
38
|
+
if (!trace.grades)
|
|
39
|
+
throw new Error(`${file} is not graded`);
|
|
40
|
+
traces.push(trace);
|
|
41
|
+
}
|
|
42
|
+
if (traces.length === 0)
|
|
43
|
+
throw new Error(`No traces for batch "${batch}"`);
|
|
44
|
+
if (new Set(traces.map((t) => t.page.sha256)).size !== 1)
|
|
45
|
+
throw new Error(`Batch "${batch}" mixes page variants`);
|
|
46
|
+
return traces;
|
|
47
|
+
}
|
|
48
|
+
const A = await loadBatch(spec, values.a);
|
|
49
|
+
const B = await loadBatch(spec, values.b);
|
|
50
|
+
if (A[0].page.sha256 === B[0].page.sha256)
|
|
51
|
+
throw new Error('Both batches ran the same page. Nothing to compare.');
|
|
52
|
+
const key = (t) => `${t.persona.id}|${t.viewport.name}|${t.seed}`;
|
|
53
|
+
const byKey = new Map(B.map((t) => [key(t), t]));
|
|
54
|
+
const pairs = A.filter((t) => byKey.has(key(t))).map((a) => ({ a, b: byKey.get(key(a)) }));
|
|
55
|
+
const passed = (t, id) => t.grades.find((g) => g.questionId === id).pass;
|
|
56
|
+
const pct = (x) => `${(x * 100).toFixed(0)}%`;
|
|
57
|
+
const pts = (x) => `${x >= 0 ? '+' : ''}${(x * 100).toFixed(0)}`;
|
|
58
|
+
function section(set) {
|
|
59
|
+
const ps = pairs.filter((p) => p.a.personaSet === set);
|
|
60
|
+
if (ps.length === 0)
|
|
61
|
+
return 'No pairs.';
|
|
62
|
+
const lines = spec.questions.map((q) => {
|
|
63
|
+
const gained = ps.filter((p) => !passed(p.a, q.id) && passed(p.b, q.id)).length;
|
|
64
|
+
const lost = ps.filter((p) => passed(p.a, q.id) && !passed(p.b, q.id)).length;
|
|
65
|
+
const d = pairedDifference(gained, lost, ps.length);
|
|
66
|
+
const rateA = ps.filter((p) => passed(p.a, q.id)).length / ps.length;
|
|
67
|
+
const wb = wilson(ps.filter((p) => passed(p.b, q.id)).length, ps.length);
|
|
68
|
+
const threshold = Number(pass[`${q.id}_rate`]);
|
|
69
|
+
const call = d.lower > 0 ? 'B BETTER' : d.upper < 0 ? 'B WORSE' : 'INSIDE THE NOISE';
|
|
70
|
+
return `| ${q.id} | ${pct(rateA)} | ${pct(wb.rate)} | ${pts(d.difference)} (${pts(d.lower)} to ${pts(d.upper)}) | ${gained} gained, ${lost} lost | ${d.p < 0.001 ? '<0.001' : d.p.toFixed(3)} | ${call} | ${pct(wb.lower)} to ${pct(wb.upper)} vs ${pct(threshold)}: ${verdict(wb, threshold).toUpperCase()} |`;
|
|
71
|
+
});
|
|
72
|
+
return [
|
|
73
|
+
`${ps.length} pairs.`,
|
|
74
|
+
'',
|
|
75
|
+
'| Question | A | B | Change in points (95% interval) | Pairs that flipped | p | Call | B against its threshold |',
|
|
76
|
+
'|---|---|---|---|---|---|---|---|',
|
|
77
|
+
...lines,
|
|
78
|
+
].join('\n');
|
|
79
|
+
}
|
|
80
|
+
const reach = (set) => {
|
|
81
|
+
const order = [];
|
|
82
|
+
for (const t of [...set].sort((x, y) => y.steps.length - x.steps.length))
|
|
83
|
+
for (const b of t.steps.flatMap((s) => s.visible))
|
|
84
|
+
if (!order.includes(b.section))
|
|
85
|
+
order.push(b.section);
|
|
86
|
+
const at = order.indexOf('scorecard');
|
|
87
|
+
return set.filter((t) => at >= 0 && order.indexOf(t.deepestSection) >= at).length / set.length;
|
|
88
|
+
};
|
|
89
|
+
const md = `# Paired comparison: ${spec.experiment}
|
|
90
|
+
|
|
91
|
+
- A (baseline): batch \`${values.a}\`, page \`${A[0].page.path}\`, sha256 \`${A[0].page.sha256.slice(0, 12)}\`, ${A[0].page.words} words
|
|
92
|
+
- B (candidate): batch \`${values.b}\`, page \`${B[0].page.path}\`, sha256 \`${B[0].page.sha256.slice(0, 12)}\`, ${B[0].page.words} words
|
|
93
|
+
- Pairs: ${pairs.length} of ${A.length} A runs and ${B.length} B runs, matched on persona, viewport, and seed
|
|
94
|
+
${A.every((t) => t.dials.calibrated) && B.every((t) => t.dials.calibrated) ? '' : '- **Dials are not calibrated.** This compares two pages for the simulated readers. It is not a prediction about real readers.\n'}
|
|
95
|
+
A change counts only when its whole 95% interval is on one side of zero. Otherwise it is inside the noise and is not a result.
|
|
96
|
+
|
|
97
|
+
## Working set
|
|
98
|
+
|
|
99
|
+
${section('working')}
|
|
100
|
+
|
|
101
|
+
## Holdout set
|
|
102
|
+
|
|
103
|
+
${section('holdout')}
|
|
104
|
+
|
|
105
|
+
## Secondary
|
|
106
|
+
|
|
107
|
+
- Readers who reached the scorecard: A ${pct(reach(A))}, B ${pct(reach(B))}
|
|
108
|
+
- Words on the page: A ${A[0].page.words}, B ${B[0].page.words}
|
|
109
|
+
|
|
110
|
+
## How to read this
|
|
111
|
+
|
|
112
|
+
The interval is for the paired difference (B minus A). The p value is an exact McNemar test on the pairs that flipped. Pairs share a persona, a viewport, and a skim seed. The model's answers and the judge's grades are not seeded, so some flips in both directions are expected even between two runs of the same page.
|
|
113
|
+
`;
|
|
114
|
+
const outDir = path.join(repoRoot, 'reports', spec.experiment);
|
|
115
|
+
await mkdir(outDir, { recursive: true });
|
|
116
|
+
const name = `compare-${values.a.replace(':', '-')}-vs-${values.b.replace(':', '-')}.md`;
|
|
117
|
+
await writeFile(path.join(outDir, name), md);
|
|
118
|
+
console.log(md);
|
|
119
|
+
console.log(`Report: reports/${spec.experiment}/${name}`);
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
// Scorer: aggregates one batch of traces into a report.
|
|
2
|
+
//
|
|
3
|
+
// node scorer/score.ts --batch <id> [--spec specs/sdd-site.spec.yaml]
|
|
4
|
+
//
|
|
5
|
+
// Writes reports/<experiment>/<batch>.md, .json, and a spot-check sheet.
|
|
6
|
+
// Reads traces. Never writes to them.
|
|
7
|
+
import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
|
|
8
|
+
import path from 'node:path';
|
|
9
|
+
import { parseArgs } from 'node:util';
|
|
10
|
+
import { parse } from 'yaml';
|
|
11
|
+
import { createRng, deriveSeed } from "../runner/rng.js";
|
|
12
|
+
import { loadSpec } from "../runner/spec.js";
|
|
13
|
+
import { validateTrace } from "../runner/trace.js";
|
|
14
|
+
import { verdict, wilson } from "./stats.js";
|
|
15
|
+
const repoRoot = path.resolve(import.meta.dirname, '..');
|
|
16
|
+
const { values } = parseArgs({
|
|
17
|
+
options: { spec: { type: 'string', default: 'specs/sdd-site.spec.yaml' }, batch: { type: 'string' }, grades: { type: 'string' } },
|
|
18
|
+
});
|
|
19
|
+
if (!values.batch) {
|
|
20
|
+
console.error('Usage: node scorer/score.ts --batch <id> [--spec file]');
|
|
21
|
+
process.exit(2);
|
|
22
|
+
}
|
|
23
|
+
const batch = values.batch;
|
|
24
|
+
const specFile = path.resolve(repoRoot, values.spec);
|
|
25
|
+
const spec = await loadSpec(specFile);
|
|
26
|
+
const pass = parse(await readFile(specFile, 'utf8')).pass;
|
|
27
|
+
const questionIds = spec.questions.map((q) => q.id);
|
|
28
|
+
const threshold = (id) => Number(pass[`${id}_rate`]);
|
|
29
|
+
// Load and check the batch.
|
|
30
|
+
const dir = path.join(repoRoot, 'traces', spec.experiment);
|
|
31
|
+
const traces = [];
|
|
32
|
+
for (const file of (await readdir(dir)).sort()) {
|
|
33
|
+
if (!file.endsWith('.json'))
|
|
34
|
+
continue;
|
|
35
|
+
const trace = JSON.parse(await readFile(path.join(dir, file), 'utf8'));
|
|
36
|
+
if (trace.batch !== batch)
|
|
37
|
+
continue;
|
|
38
|
+
validateTrace(trace, questionIds);
|
|
39
|
+
if (values.grades) {
|
|
40
|
+
// Use a regrade instead of the grades stored in the trace. The trace file is not changed.
|
|
41
|
+
const regradeFile = path.join(repoRoot, 'grades', spec.experiment, batch, values.grades, file);
|
|
42
|
+
const regrade = JSON.parse(await readFile(regradeFile, 'utf8'));
|
|
43
|
+
trace.grades = regrade.grades;
|
|
44
|
+
trace.llmCalls = [...trace.llmCalls, ...regrade.llmCalls];
|
|
45
|
+
trace.cost.usd += regrade.costUsd;
|
|
46
|
+
}
|
|
47
|
+
if (!trace.grades)
|
|
48
|
+
throw new Error(`${file} is not graded`);
|
|
49
|
+
traces.push(trace);
|
|
50
|
+
}
|
|
51
|
+
if (traces.length === 0)
|
|
52
|
+
throw new Error(`No traces found for batch "${batch}"`);
|
|
53
|
+
const pageHashes = new Set(traces.map((t) => t.page.sha256));
|
|
54
|
+
if (pageHashes.size !== 1)
|
|
55
|
+
throw new Error(`Batch mixes ${pageHashes.size} page variants. A batch scores one page.`);
|
|
56
|
+
const passed = (t, id) => t.grades.find((g) => g.questionId === id).pass;
|
|
57
|
+
const rate = (set, id) => wilson(set.filter((t) => passed(t, id)).length, set.length);
|
|
58
|
+
const working = traces.filter((t) => t.personaSet === 'working');
|
|
59
|
+
const holdout = traces.filter((t) => t.personaSet === 'holdout');
|
|
60
|
+
const rows = (set) => questionIds.map((id) => {
|
|
61
|
+
const interval = rate(set, id);
|
|
62
|
+
return { id, threshold: threshold(id), interval, verdict: verdict(interval, threshold(id)) };
|
|
63
|
+
});
|
|
64
|
+
const workingRows = rows(working);
|
|
65
|
+
const holdoutRows = rows(holdout);
|
|
66
|
+
const workingMet = workingRows.every((r) => r.verdict === 'met');
|
|
67
|
+
const holdoutMet = holdout.length > 0 && holdoutRows.every((r) => r.verdict === 'met');
|
|
68
|
+
const overall = workingMet && holdoutMet;
|
|
69
|
+
// Where readers stopped. Section order comes from the page as readers saw it.
|
|
70
|
+
const sectionOrder = [];
|
|
71
|
+
for (const t of [...traces].sort((a, b) => b.steps.length - a.steps.length)) {
|
|
72
|
+
for (const b of t.steps.flatMap((s) => s.visible))
|
|
73
|
+
if (!sectionOrder.includes(b.section))
|
|
74
|
+
sectionOrder.push(b.section);
|
|
75
|
+
}
|
|
76
|
+
const reached = (t, section) => {
|
|
77
|
+
const at = sectionOrder.indexOf(section);
|
|
78
|
+
return at >= 0 && sectionOrder.indexOf(t.deepestSection) >= at;
|
|
79
|
+
};
|
|
80
|
+
// Which rubric elements were missed.
|
|
81
|
+
const missed = spec.questions.flatMap((q) => q.rubric.required.map((element, i) => ({
|
|
82
|
+
question: q.id,
|
|
83
|
+
element,
|
|
84
|
+
working: wilson(working.filter((t) => !t.grades.find((g) => g.questionId === q.id).requiredMet[i]).length, working.length),
|
|
85
|
+
holdout: wilson(holdout.filter((t) => !t.grades.find((g) => g.questionId === q.id).requiredMet[i]).length, holdout.length),
|
|
86
|
+
})));
|
|
87
|
+
const calls = traces.flatMap((t) => t.llmCalls);
|
|
88
|
+
const costBy = (role) => calls.filter((c) => c.request.role === role).reduce((s, c) => s + c.response.costUsd, 0);
|
|
89
|
+
const modelOf = (role) => calls.find((c) => c.request.role === role)?.response.model ?? 'none';
|
|
90
|
+
const totalCost = traces.reduce((s, t) => s + t.cost.usd, 0);
|
|
91
|
+
const calibrated = traces.every((t) => t.dials.calibrated);
|
|
92
|
+
const first = traces[0];
|
|
93
|
+
// ---------- report ----------
|
|
94
|
+
const pct = (x) => (Number.isNaN(x) ? 'n/a' : `${(x * 100).toFixed(0)}%`);
|
|
95
|
+
const ci = (i) => `${pct(i.lower)} to ${pct(i.upper)}`;
|
|
96
|
+
const cell = (i) => `${pct(i.rate)} (${i.passed}/${i.n})`;
|
|
97
|
+
const table = (head, body) => [`| ${head.join(' | ')} |`, `|${head.map(() => '---').join('|')}|`, ...body.map((r) => `| ${r.join(' | ')} |`)].join('\n');
|
|
98
|
+
const verdictTable = (set) => table(['Question', 'Passed', '95% interval', 'Threshold', 'Verdict'], set.map((r) => [r.id, cell(r.interval), ci(r.interval), pct(r.threshold), r.verdict.toUpperCase()]));
|
|
99
|
+
const groupTable = (label, groups) => table([label, 'Runs', ...questionIds], [...groups].map(([name, set]) => [name, String(set.length), ...questionIds.map((id) => `${cell(rate(set, id))}, ${ci(rate(set, id))}`)]));
|
|
100
|
+
const groupBy = (set, f) => {
|
|
101
|
+
const m = new Map();
|
|
102
|
+
for (const t of set)
|
|
103
|
+
m.set(f(t), [...(m.get(f(t)) ?? []), t]);
|
|
104
|
+
return m;
|
|
105
|
+
};
|
|
106
|
+
const noisy = [...workingRows.map((r) => `working ${r.id}`), ...holdoutRows.map((r) => `holdout ${r.id}`)].filter((_, i) => [...workingRows, ...holdoutRows][i].verdict === 'inside the noise');
|
|
107
|
+
const viewportNames = [...new Set(traces.map((t) => t.viewport.name))];
|
|
108
|
+
const scorecardReach = wilson(working.filter((t) => reached(t, 'scorecard')).length, working.length);
|
|
109
|
+
const md = `# Score report: ${spec.experiment}, batch "${batch}"
|
|
110
|
+
|
|
111
|
+
**Overall: ${overall ? 'PASS' : 'NOT PASSED'}.** ${overall
|
|
112
|
+
? 'Every threshold is cleared by the lower bound of its interval, on the working set and on the holdout set.'
|
|
113
|
+
: 'A threshold counts as met only when the lower bound of its 95% interval clears it, on both the working set and the holdout set.'}
|
|
114
|
+
|
|
115
|
+
${calibrated ? '' : '**These dials are not calibrated.** No real reader has been measured yet, so these rates say how the simulated readers did, not how real developers would. Do not publish them as predictions.\n'}
|
|
116
|
+
- Page: \`${first.page.path}\`, sha256 \`${first.page.sha256.slice(0, 12)}\`, ${first.page.words} visible words
|
|
117
|
+
- Runs: ${traces.length} (${working.length} working, ${holdout.length} holdout), viewports: ${viewportNames.join(', ')}
|
|
118
|
+
- Time budget: ${first.dials.timeBudgetSeconds} simulated seconds. Skim scale ${first.dials.skimScale}, recall decay ${first.dials.recallDecayPerSecond} per second.
|
|
119
|
+
${values.grades ? `- Grades: regrade "${values.grades}". Cost includes the original grading and the regrade.\n` : ''}- Models: persona \`${modelOf('persona')}\`, judge \`${modelOf('judge')}\`
|
|
120
|
+
- Cost at list prices: $${totalCost.toFixed(2)} (persona $${costBy('persona').toFixed(2)}, judge $${costBy('judge').toFixed(2)})
|
|
121
|
+
|
|
122
|
+
## Working set
|
|
123
|
+
|
|
124
|
+
${verdictTable(workingRows)}
|
|
125
|
+
|
|
126
|
+
## Holdout set
|
|
127
|
+
|
|
128
|
+
Kept separate. The revising agent never sees these personas.
|
|
129
|
+
|
|
130
|
+
${holdout.length ? verdictTable(holdoutRows) : 'No holdout runs in this batch.'}
|
|
131
|
+
|
|
132
|
+
Thresholds hold on the holdout set: **${holdout.length === 0 ? 'not run' : holdoutMet ? 'yes' : 'no'}**.${holdout.length === 0
|
|
133
|
+
? ''
|
|
134
|
+
: ` Questions where holdout and working get a different verdict: ${workingRows.filter((r, i) => r.verdict !== holdoutRows[i].verdict).map((r) => r.id).join(', ') || 'none'}.`}
|
|
135
|
+
|
|
136
|
+
## What is inside the noise
|
|
137
|
+
|
|
138
|
+
${noisy.length
|
|
139
|
+
? `For these, the interval straddles the threshold, so this batch cannot say whether the threshold is met: ${noisy.join(', ')}. More runs would narrow the interval.`
|
|
140
|
+
: 'Nothing. Every interval sits clearly above or clearly below its threshold.'}
|
|
141
|
+
|
|
142
|
+
The intervals treat every run as independent. Runs share personas, and the same persona tends to succeed or fail in the same way, so the true uncertainty is somewhat wider than shown.
|
|
143
|
+
|
|
144
|
+
## By viewport (working set)
|
|
145
|
+
|
|
146
|
+
${groupTable('Viewport', groupBy(working, (t) => t.viewport.name))}
|
|
147
|
+
|
|
148
|
+
## By persona
|
|
149
|
+
|
|
150
|
+
${groupTable('Persona', groupBy(traces, (t) => `${t.persona.id}${t.personaSet === 'holdout' ? ' (holdout)' : ''}`))}
|
|
151
|
+
|
|
152
|
+
## Where readers stopped
|
|
153
|
+
|
|
154
|
+
Share of runs whose reader got at least as far as each section before the time ran out.
|
|
155
|
+
|
|
156
|
+
${table(['Section', ...viewportNames.map((v) => `${v} (working)`), 'holdout'], sectionOrder.map((s) => [
|
|
157
|
+
s,
|
|
158
|
+
...viewportNames.map((v) => {
|
|
159
|
+
const set = working.filter((t) => t.viewport.name === v);
|
|
160
|
+
return pct(set.filter((t) => reached(t, s)).length / set.length);
|
|
161
|
+
}),
|
|
162
|
+
holdout.length ? pct(holdout.filter((t) => reached(t, s)).length / holdout.length) : 'n/a',
|
|
163
|
+
]))}
|
|
164
|
+
|
|
165
|
+
Sections that no reader reached do not appear above.
|
|
166
|
+
|
|
167
|
+
## Rubric elements missed most
|
|
168
|
+
|
|
169
|
+
Share of answers that did not state each required element.
|
|
170
|
+
|
|
171
|
+
${table(['Question', 'Required element', 'Missed (working)', 'Missed (holdout)'], [...missed].sort((a, b) => b.working.rate - a.working.rate).map((m) => [m.question, m.element, cell(m.working), holdout.length ? cell(m.holdout) : 'n/a']))}
|
|
172
|
+
|
|
173
|
+
## Secondary measures (reported, not gating)
|
|
174
|
+
|
|
175
|
+
- Readers who reached the scorecard section (working): ${cell(scorecardReach)}, interval ${ci(scorecardReach)}
|
|
176
|
+
- Words on the page: ${first.page.words}
|
|
177
|
+
- Skeptic personas whose main objection was answered: not measured. Reader mode does not ask this yet.
|
|
178
|
+
`;
|
|
179
|
+
const outDir = path.join(repoRoot, 'reports', spec.experiment);
|
|
180
|
+
const outName = values.grades ? `${batch}-judge-${values.grades}` : batch;
|
|
181
|
+
await mkdir(outDir, { recursive: true });
|
|
182
|
+
await writeFile(path.join(outDir, `${outName}.md`), md);
|
|
183
|
+
await writeFile(path.join(outDir, `${outName}.json`), JSON.stringify({ experiment: spec.experiment, batch, page: first.page, runs: traces.length, calibrated, overall, working: workingRows, holdout: holdoutRows, missed, sectionOrder, scorecardReach, costUsd: totalCost }, null, 2) + '\n');
|
|
184
|
+
// Spot-check sheet: a seeded 10 percent sample of graded answers for the owner.
|
|
185
|
+
const rng = createRng(deriveSeed('spot-check', batch));
|
|
186
|
+
const csv = (s) => `"${s.replaceAll('"', '""')}"`;
|
|
187
|
+
const sample = traces
|
|
188
|
+
.flatMap((t) => t.grades.map((g) => ({ t, g })))
|
|
189
|
+
.filter(() => rng() < 0.1)
|
|
190
|
+
.map(({ t, g }) => {
|
|
191
|
+
const q = spec.questions.find((x) => x.id === g.questionId);
|
|
192
|
+
return [
|
|
193
|
+
t.runId, q.id, q.ask, t.answers.find((a) => a.id === q.id).answer,
|
|
194
|
+
q.rubric.required.map((r, i) => `${g.requiredMet[i] ? 'MET' : 'NOT MET'}: ${r}`).join(' / '),
|
|
195
|
+
g.pass ? 'pass' : 'fail', g.note, '', '',
|
|
196
|
+
].map(csv).join(',');
|
|
197
|
+
});
|
|
198
|
+
await writeFile(path.join(outDir, `${outName}-spot-check.csv`), ['run,question,asked,answer,judge_on_required_elements,judge_grade,judge_note,chad_grade,chad_comment', ...sample].join('\n') + '\n');
|
|
199
|
+
console.log(md.split('\n## By viewport')[0]);
|
|
200
|
+
console.log(`Report: reports/${spec.experiment}/${outName}.md`);
|
|
201
|
+
console.log(`Spot-check sheet: reports/${spec.experiment}/${outName}-spot-check.csv (${sample.length} answers)`);
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// Compares a second reviewer's grades with the judge's on a spot-check sheet.
|
|
2
|
+
// node scorer/spot-check-compare.ts <sheet.csv> <review.json>
|
|
3
|
+
// review.json: { reviewer, grades: { "<question>": "row:TF row:TT ..." } }, one letter per required element.
|
|
4
|
+
import { readFileSync } from 'node:fs';
|
|
5
|
+
import { parseCsv } from "./spot-check-view.js";
|
|
6
|
+
const [sheet, reviewFile] = process.argv.slice(2);
|
|
7
|
+
const rows = parseCsv(readFileSync(sheet, 'utf8')).slice(1);
|
|
8
|
+
const review = JSON.parse(readFileSync(reviewFile, 'utf8'));
|
|
9
|
+
console.log(`Reviewer: ${review.reviewer}\n`);
|
|
10
|
+
const disagreements = [];
|
|
11
|
+
let allSame = 0, all = 0;
|
|
12
|
+
for (const [question, text] of Object.entries(review.grades)) {
|
|
13
|
+
const mine = new Map(text.trim().split(/\s+/).map((x) => x.split(':')));
|
|
14
|
+
const sheetRows = rows.map((r, i) => ({ r, n: i + 1 })).filter((x) => x.r[1] === question);
|
|
15
|
+
if (sheetRows.length !== mine.size)
|
|
16
|
+
throw new Error(`${question}: sheet has ${sheetRows.length} rows, review has ${mine.size}`);
|
|
17
|
+
let passSame = 0, judgePass = 0, minePass = 0;
|
|
18
|
+
const elementSame = [], elements = [];
|
|
19
|
+
for (const { r, n } of sheetRows) {
|
|
20
|
+
const judge = r[4].split(' / ').map((e) => e.startsWith('MET'));
|
|
21
|
+
r[4].split(' / ').forEach((e, i) => (elements[i] = e.replace(/^(NOT )?MET: /, '')));
|
|
22
|
+
const m = [...mine.get(String(n))].map((c) => c === 'T');
|
|
23
|
+
if (m.length !== judge.length)
|
|
24
|
+
throw new Error(`row ${n}: element count differs`);
|
|
25
|
+
judge.forEach((j, i) => (elementSame[i] = (elementSame[i] ?? 0) + (j === m[i] ? 1 : 0)));
|
|
26
|
+
const jp = judge.every(Boolean), mp = m.every(Boolean);
|
|
27
|
+
judgePass += +jp;
|
|
28
|
+
minePass += +mp;
|
|
29
|
+
passSame += +(jp === mp);
|
|
30
|
+
if (jp !== mp)
|
|
31
|
+
disagreements.push(`row ${n} ${question}: judge ${jp ? 'pass' : 'fail'}, reviewer ${mp ? 'pass' : 'fail'} | ${r[6]} | ${r[3].slice(0, 160)}`);
|
|
32
|
+
}
|
|
33
|
+
const n = sheetRows.length;
|
|
34
|
+
all += n;
|
|
35
|
+
allSame += passSame;
|
|
36
|
+
console.log(`${question}: ${n} answers, same pass/fail on ${passSame} (${Math.round((100 * passSame) / n)}%). Judge passed ${judgePass}, reviewer passed ${minePass}.`);
|
|
37
|
+
elements.forEach((e, i) => console.log(` element "${e}": agree on ${elementSame[i]}/${n}`));
|
|
38
|
+
}
|
|
39
|
+
console.log(`\nOverall: same pass/fail on ${allSame} of ${all} (${Math.round((100 * allSame) / all)}%)\n\nDisagreements:`);
|
|
40
|
+
for (const d of disagreements)
|
|
41
|
+
console.log(`- ${d}`);
|