tick-bench-cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CITATION.cff +21 -0
- package/LICENSE +21 -0
- package/README.md +187 -0
- package/docs/agent-adapters.md +31 -0
- package/docs/banner.svg +59 -0
- package/docs/connect-your-table.md +104 -0
- package/docs/metrics.md +28 -0
- package/docs/report-screenshot.png +0 -0
- package/docs/task-authoring.md +28 -0
- package/examples/my-table-adapter.mjs +57 -0
- package/examples/walkthrough/README.md +114 -0
- package/examples/walkthrough/adapter.mjs +35 -0
- package/examples/walkthrough/my-table.mjs +49 -0
- package/examples/walkthrough/sample-results/report-stream-grid.html +23 -0
- package/examples/walkthrough/sample-results/report-stream-grid.json +74 -0
- package/package.json +61 -0
- package/schemas/task.schema.json +24 -0
- package/schemas/trace.schema.json +28 -0
- package/src/cli.mjs +129 -0
- package/src/index.d.ts +77 -0
- package/src/index.mjs +7 -0
- package/src/oracles/a11y.mjs +31 -0
- package/src/oracles/runtime.mjs +93 -0
- package/src/oracles/security.mjs +24 -0
- package/src/report/aggregate.mjs +17 -0
- package/src/report/html.mjs +56 -0
- package/src/report/verdict.mjs +16 -0
- package/src/trace/generate.mjs +29 -0
- package/tasks/stream-grid/oracle.config.json +14 -0
- package/tasks/stream-grid/reference/grid.mjs +29 -0
- package/tasks/stream-grid/spec.md +9 -0
- package/tasks/stream-grid/submissions/claude-fable-5/PROVENANCE.md +10 -0
- package/tasks/stream-grid/submissions/claude-fable-5/grid.mjs +72 -0
- package/tasks/stream-grid/submissions/naive-baseline/PROVENANCE.md +6 -0
- package/tasks/stream-grid/submissions/naive-baseline/grid.mjs +32 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// Accessibility oracle: renders the grid with initial data, runs axe-core WCAG 2.1 A/AA rules.
|
|
2
|
+
import { readFileSync, writeFileSync, mkdtempSync } from 'node:fs';
|
|
3
|
+
import { join, resolve } from 'node:path';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { pathToFileURL } from 'node:url';
|
|
6
|
+
import { createRequire } from 'node:module';
|
|
7
|
+
import { chromium } from 'playwright-core';
|
|
8
|
+
const require = createRequire(import.meta.url);
|
|
9
|
+
|
|
10
|
+
export async function runA11yOracle({ implPath, trace, warmupTicks = 400 }) {
|
|
11
|
+
const implUrl = pathToFileURL(resolve(implPath)).href;
|
|
12
|
+
const axeSrc = readFileSync(require.resolve('axe-core/axe.min.js'), 'utf8');
|
|
13
|
+
const pageFile = join(mkdtempSync(join(tmpdir(), 'tickbench-')), 'page.html');
|
|
14
|
+
writeFileSync(pageFile, '<!doctype html><html lang="en"><head><meta charset="utf-8"><title>TickBench</title></head><body><div id="grid"></div></body></html>');
|
|
15
|
+
const browser = await chromium.launch({ headless: true, args: ['--no-sandbox', '--disable-gpu', '--allow-file-access-from-files'] });
|
|
16
|
+
try {
|
|
17
|
+
const page = await browser.newPage();
|
|
18
|
+
await page.goto(pathToFileURL(pageFile).href);
|
|
19
|
+
await page.evaluate(async ({ implUrl, trace, warmupTicks }) => {
|
|
20
|
+
const mod = await import(implUrl);
|
|
21
|
+
const grid = mod.createGrid(document.getElementById('grid'), trace.symbols, trace.cols);
|
|
22
|
+
for (const tk of trace.ticks.slice(0, warmupTicks)) grid.applyTick(tk);
|
|
23
|
+
await new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)));
|
|
24
|
+
}, { implUrl, trace, warmupTicks });
|
|
25
|
+
await page.addScriptTag({ content: axeSrc });
|
|
26
|
+
return await page.evaluate(async () => {
|
|
27
|
+
const r = await axe.run(document, { runOnly: { type: 'tag', values: ['wcag2a', 'wcag2aa', 'wcag21aa'] } });
|
|
28
|
+
return r.violations.map(v => ({ id: v.id, impact: v.impact, nodes: v.nodes.length }));
|
|
29
|
+
});
|
|
30
|
+
} finally { await browser.close(); }
|
|
31
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// Runtime oracle: replays a tick trace against a submission inside headless Chromium,
|
|
2
|
+
// sampling rendered cells every animation frame. Metric definitions: docs/metrics.md.
|
|
3
|
+
import { writeFileSync, mkdtempSync } from 'node:fs';
|
|
4
|
+
import { join, resolve } from 'node:path';
|
|
5
|
+
import { tmpdir } from 'node:os';
|
|
6
|
+
import { pathToFileURL } from 'node:url';
|
|
7
|
+
import { chromium } from 'playwright-core';
|
|
8
|
+
|
|
9
|
+
// Submissions are imported as real file:// ES modules, so multi-file components with
|
|
10
|
+
// relative imports work exactly like in a bundler-free browser project.
|
|
11
|
+
export async function runRuntimeOracle({ implPath, trace, quiescenceMs = 600 }) {
|
|
12
|
+
const implUrl = pathToFileURL(resolve(implPath)).href;
|
|
13
|
+
const pageFile = join(mkdtempSync(join(tmpdir(), 'tickbench-')), 'page.html');
|
|
14
|
+
writeFileSync(pageFile, '<!doctype html><html lang="en"><head><meta charset="utf-8"><title>TickBench</title></head><body><div id="grid"></div></body></html>');
|
|
15
|
+
const browser = await chromium.launch({ headless: true, args: ['--no-sandbox', '--disable-gpu', '--allow-file-access-from-files'] });
|
|
16
|
+
try {
|
|
17
|
+
const page = await (await browser.newContext({ viewport: { width: 1280, height: 2400 } })).newPage();
|
|
18
|
+
await page.goto(pathToFileURL(pageFile).href);
|
|
19
|
+
return await page.evaluate(async ({ implUrl, trace, quiescenceMs }) => {
|
|
20
|
+
const mod = await import(implUrl);
|
|
21
|
+
const grid = mod.createGrid(document.getElementById('grid'), trace.symbols, trace.cols);
|
|
22
|
+
|
|
23
|
+
// Independent expected model, derived from the task spec (never from a submission).
|
|
24
|
+
const fmtP = new Intl.NumberFormat('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 });
|
|
25
|
+
const fmtQ = new Intl.NumberFormat('en-US');
|
|
26
|
+
const expected = new Map();
|
|
27
|
+
const catchUp = [];
|
|
28
|
+
const frames = [];
|
|
29
|
+
let staleCellFrames = 0, cellFrames = 0;
|
|
30
|
+
const key = t => t.sym + '|' + t.col;
|
|
31
|
+
const numOf = txt => { const n = Number(String(txt).replace(/,/g, '')); return Number.isFinite(n) ? n : NaN; };
|
|
32
|
+
const expNum = e => e.col === 'qty' ? e.v : e.v / 100;
|
|
33
|
+
|
|
34
|
+
const t0 = performance.now();
|
|
35
|
+
let i = 0;
|
|
36
|
+
await new Promise(resolve => {
|
|
37
|
+
function pump() {
|
|
38
|
+
const now = performance.now() - t0;
|
|
39
|
+
while (i < trace.ticks.length && trace.ticks[i].t <= now) {
|
|
40
|
+
const tk = trace.ticks[i];
|
|
41
|
+
const k = key(tk);
|
|
42
|
+
const e = expected.get(k) || { ver: 0, matchedVer: 0, col: tk.col };
|
|
43
|
+
e.v = tk.v; e.ver++; e.arrival = performance.now(); e.col = tk.col;
|
|
44
|
+
expected.set(k, e);
|
|
45
|
+
try { grid.applyTick(tk); } catch (err) { window.__implError = String(err); }
|
|
46
|
+
i++;
|
|
47
|
+
}
|
|
48
|
+
if (i < trace.ticks.length) setTimeout(pump, 0);
|
|
49
|
+
else setTimeout(resolve, quiescenceMs);
|
|
50
|
+
}
|
|
51
|
+
pump();
|
|
52
|
+
let last = performance.now();
|
|
53
|
+
function sample(ts) {
|
|
54
|
+
frames.push(ts - last); last = ts;
|
|
55
|
+
for (const td of document.querySelectorAll('td[data-sym]')) {
|
|
56
|
+
const k = td.dataset.sym + '|' + td.dataset.col;
|
|
57
|
+
const e = expected.get(k);
|
|
58
|
+
if (!e) continue;
|
|
59
|
+
cellFrames++;
|
|
60
|
+
const match = Math.abs(numOf(td.textContent) - expNum(e)) < 0.005;
|
|
61
|
+
if (match) {
|
|
62
|
+
if (e.matchedVer < e.ver) { catchUp.push(performance.now() - e.arrival); e.matchedVer = e.ver; }
|
|
63
|
+
} else staleCellFrames++;
|
|
64
|
+
}
|
|
65
|
+
if (i < trace.ticks.length || performance.now() - t0 < trace.durationMs + quiescenceMs - 100) requestAnimationFrame(sample);
|
|
66
|
+
}
|
|
67
|
+
requestAnimationFrame(sample);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
let valueErrors = 0, formatErrors = 0, checked = 0;
|
|
71
|
+
for (const td of document.querySelectorAll('td[data-sym]')) {
|
|
72
|
+
const k = td.dataset.sym + '|' + td.dataset.col;
|
|
73
|
+
const e = expected.get(k);
|
|
74
|
+
if (!e) continue;
|
|
75
|
+
checked++;
|
|
76
|
+
const want = e.col === 'qty' ? fmtQ.format(e.v) : fmtP.format(e.v / 100);
|
|
77
|
+
if (Math.abs(numOf(td.textContent) - expNum(e)) >= 0.005) valueErrors++;
|
|
78
|
+
else if (td.textContent !== want) formatErrors++;
|
|
79
|
+
}
|
|
80
|
+
const pct = (arr, p) => { if (!arr.length) return null; const s = [...arr].sort((a, b) => a - b); return s[Math.min(s.length - 1, Math.floor(p * s.length))]; };
|
|
81
|
+
const longFrames = frames.filter(d => d > 33.4).length;
|
|
82
|
+
return {
|
|
83
|
+
ticksDispatched: i, cellsChecked: checked,
|
|
84
|
+
catchUpMs: { p50: pct(catchUp, .5), p95: pct(catchUp, .95), p99: pct(catchUp, .99), n: catchUp.length },
|
|
85
|
+
staleCellFramePct: cellFrames ? +(100 * staleCellFrames / cellFrames).toFixed(2) : null,
|
|
86
|
+
frames: frames.length, longFramePct: frames.length ? +(100 * longFrames / frames.length).toFixed(2) : null,
|
|
87
|
+
worstFrameMs: frames.length ? +Math.max(...frames).toFixed(1) : null,
|
|
88
|
+
finalValueErrors: valueErrors, finalFormatErrors: formatErrors,
|
|
89
|
+
xssTriggered: window.__xss === 1, implError: window.__implError || null
|
|
90
|
+
};
|
|
91
|
+
}, { implUrl, trace, quiescenceMs });
|
|
92
|
+
} finally { await browser.close(); }
|
|
93
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Static security oracle: flags client-side injection sinks and monetary-precision hazards.
|
|
2
|
+
// Deeper scanning (Semgrep/CodeQL rulesets) can be layered on top; dynamic XSS detection lives in
|
|
3
|
+
// the runtime oracle (canary symbol).
|
|
4
|
+
import { readFileSync } from 'node:fs';
|
|
5
|
+
|
|
6
|
+
export const RULES = [
|
|
7
|
+
{ id: 'js-inner-html', re: /\.innerHTML\s*=/g, severity: 'high', why: 'innerHTML assignment (XSS sink)' },
|
|
8
|
+
{ id: 'js-insert-html', re: /insertAdjacentHTML\s*\(/g, severity: 'high', why: 'insertAdjacentHTML (XSS sink)' },
|
|
9
|
+
{ id: 'js-doc-write', re: /document\.write\s*\(/g, severity: 'high', why: 'document.write (XSS sink)' },
|
|
10
|
+
{ id: 'js-eval', re: /\beval\s*\(/g, severity: 'high', why: 'eval' },
|
|
11
|
+
{ id: 'react-dsih', re: /dangerouslySetInnerHTML/g, severity: 'high', why: 'dangerouslySetInnerHTML' },
|
|
12
|
+
{ id: 'float-tofixed', re: /\.toFixed\s*\(\s*2\s*\)/g, severity: 'medium', why: 'float→toFixed(2) monetary formatting (precision hazard)' }
|
|
13
|
+
];
|
|
14
|
+
|
|
15
|
+
export function scanSecuritySource(src) {
|
|
16
|
+
const findings = [];
|
|
17
|
+
for (const r of RULES) {
|
|
18
|
+
const m = src.match(r.re);
|
|
19
|
+
if (m) findings.push({ id: r.id, severity: r.severity, count: m.length, why: r.why });
|
|
20
|
+
}
|
|
21
|
+
return findings;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function scanSecurity(path) { return scanSecuritySource(readFileSync(path, 'utf8')); }
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export function summaryTable(results) {
|
|
2
|
+
const names = Object.keys(results);
|
|
3
|
+
const row = (label, f) => label.padEnd(34) + names.map(n => String(f(results[n])).padStart(16)).join('');
|
|
4
|
+
const lines = [' '.repeat(34) + names.map(n => n.padStart(16)).join('')];
|
|
5
|
+
lines.push(row('catch-up latency p50 (ms)', x => x.runtime.catchUpMs.p50?.toFixed(1)));
|
|
6
|
+
lines.push(row('catch-up latency p95 (ms)', x => x.runtime.catchUpMs.p95?.toFixed(1)));
|
|
7
|
+
lines.push(row('stale cell-frames (%)', x => x.runtime.staleCellFramePct));
|
|
8
|
+
lines.push(row('long frames >33ms (%)', x => x.runtime.longFramePct));
|
|
9
|
+
lines.push(row('worst frame (ms)', x => x.runtime.worstFrameMs));
|
|
10
|
+
lines.push(row('final value errors', x => x.runtime.finalValueErrors));
|
|
11
|
+
lines.push(row('final format errors', x => x.runtime.finalFormatErrors));
|
|
12
|
+
lines.push(row('XSS triggered (dynamic)', x => x.runtime.xssTriggered));
|
|
13
|
+
lines.push(row('security findings (high)', x => x.security.filter(f => f.severity === 'high').length));
|
|
14
|
+
lines.push(row('axe violations', x => x.a11y.length));
|
|
15
|
+
lines.push(row('REGULATED-GRADE PASS', x => x.verdict.regulated_grade_pass));
|
|
16
|
+
return lines.join('\n');
|
|
17
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
// Self-contained HTML report: verdict cards + metric comparison table.
|
|
2
|
+
const esc = s => String(s).replace(/[&<>"]/g, c => ({'&':'&','<':'<','>':'>','"':'"'}[c]));
|
|
3
|
+
|
|
4
|
+
export function renderHtmlReport(report) {
|
|
5
|
+
const names = Object.keys(report.results);
|
|
6
|
+
const rows = [
|
|
7
|
+
['Catch-up latency p50 (ms)', x => x.runtime.catchUpMs.p50?.toFixed(1)],
|
|
8
|
+
['Catch-up latency p95 (ms)', x => x.runtime.catchUpMs.p95?.toFixed(1)],
|
|
9
|
+
['Catch-up latency p99 (ms)', x => x.runtime.catchUpMs.p99?.toFixed(1)],
|
|
10
|
+
['Stale cell-frames (%)', x => x.runtime.staleCellFramePct],
|
|
11
|
+
['Long frames >33ms (%)', x => x.runtime.longFramePct],
|
|
12
|
+
['Worst frame (ms)', x => x.runtime.worstFrameMs],
|
|
13
|
+
['Final value errors', x => x.runtime.finalValueErrors],
|
|
14
|
+
['Final format errors', x => x.runtime.finalFormatErrors],
|
|
15
|
+
['XSS triggered (dynamic)', x => x.runtime.xssTriggered],
|
|
16
|
+
['Security findings (high)', x => x.security.filter(f => f.severity === 'high').length],
|
|
17
|
+
['WCAG violations (axe)', x => x.a11y.length]
|
|
18
|
+
];
|
|
19
|
+
const gateLabels = {
|
|
20
|
+
functional_final_state: 'Final state correct', staleness: 'Staleness within budget',
|
|
21
|
+
frame_stability: 'Frame stability', security: 'Security', accessibility: 'Accessibility',
|
|
22
|
+
no_runtime_error: 'No runtime errors'
|
|
23
|
+
};
|
|
24
|
+
const cards = names.map(n => {
|
|
25
|
+
const v = report.results[n].verdict;
|
|
26
|
+
const pass = v.regulated_grade_pass;
|
|
27
|
+
const gates = Object.entries(gateLabels).map(([k, label]) =>
|
|
28
|
+
`<li class="${v[k] ? 'ok' : 'bad'}">${v[k] ? '✓' : '✗'} ${esc(label)}</li>`).join('');
|
|
29
|
+
return `<div class="card ${pass ? 'pass' : 'fail'}"><h2>${esc(n)}</h2>
|
|
30
|
+
<div class="badge">${pass ? 'PASS' : 'FAIL'}</div><ul>${gates}</ul></div>`;
|
|
31
|
+
}).join('');
|
|
32
|
+
const table = `<table><thead><tr><th>Metric</th>${names.map(n => `<th>${esc(n)}</th>`).join('')}</tr></thead><tbody>${
|
|
33
|
+
rows.map(([label, f]) => `<tr><td>${esc(label)}</td>${names.map(n => `<td>${esc(f(report.results[n]) ?? '—')}</td>`).join('')}</tr>`).join('')
|
|
34
|
+
}</tbody></table>`;
|
|
35
|
+
return `<!doctype html><html lang="en"><head><meta charset="utf-8"><title>TickBench report — ${esc(report.task)}</title>
|
|
36
|
+
<style>
|
|
37
|
+
body{background:#0b1220;color:#e2e8f0;font:15px/1.5 ui-sans-serif,system-ui,'Segoe UI',sans-serif;margin:0;padding:40px}
|
|
38
|
+
h1{font-size:26px} .sub{color:#94a3b8;margin-bottom:28px}
|
|
39
|
+
.cards{display:flex;gap:16px;flex-wrap:wrap;margin-bottom:32px}
|
|
40
|
+
.card{background:#0d1626;border:1px solid #24405c;border-radius:10px;padding:18px 22px;min-width:220px}
|
|
41
|
+
.card h2{margin:0 0 8px;font-size:17px}
|
|
42
|
+
.card.pass{border-color:#22d3a5}.card.fail{border-color:#f87171}
|
|
43
|
+
.badge{display:inline-block;font-weight:700;font-size:13px;border-radius:6px;padding:2px 10px;margin-bottom:10px}
|
|
44
|
+
.pass .badge{background:#0f2e26;color:#22d3a5}.fail .badge{background:#301722;color:#f87171}
|
|
45
|
+
ul{list-style:none;margin:0;padding:0;font-size:13.5px} li{padding:1.5px 0} li.ok{color:#86efac} li.bad{color:#fca5a5}
|
|
46
|
+
table{border-collapse:collapse;width:100%;max-width:900px;font-variant-numeric:tabular-nums}
|
|
47
|
+
th,td{text-align:left;padding:8px 14px;border-bottom:1px solid #1e3a52} th{color:#94a3b8;font-weight:600}
|
|
48
|
+
tr:hover td{background:#0d1626}
|
|
49
|
+
.meta{color:#64748b;font-size:13px;margin-top:26px}
|
|
50
|
+
</style></head><body>
|
|
51
|
+
<h1>TickBench report — <code>${esc(report.task)}</code></h1>
|
|
52
|
+
<div class="sub">trace <code>${esc(report.trace)}</code> · seed ${esc(report.traceSeed)} · ${esc(report.generated)} · Node ${esc(report.node)}</div>
|
|
53
|
+
<div class="cards">${cards}</div>${table}
|
|
54
|
+
<div class="meta">Regulated-grade thresholds come from the task's <code>oracle.config.json</code>. Metric definitions: docs/metrics.md.</div>
|
|
55
|
+
</body></html>`;
|
|
56
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// Config-driven regulated-grade verdict. Thresholds come from the task's oracle.config.json
|
|
2
|
+
// so that acceptance criteria are part of the task definition, not the harness.
|
|
3
|
+
export function computeVerdict({ runtime, a11y, security }, thresholds) {
|
|
4
|
+
const highSec = security.filter(f => f.severity === 'high').length;
|
|
5
|
+
const v = {
|
|
6
|
+
functional_final_state: runtime.finalValueErrors === 0 && runtime.finalFormatErrors <= (thresholds.finalErrorsMax ?? 0),
|
|
7
|
+
staleness: runtime.catchUpMs.p95 !== null && runtime.catchUpMs.p95 < thresholds.catchUpP95Ms
|
|
8
|
+
&& runtime.staleCellFramePct !== null && runtime.staleCellFramePct < thresholds.staleCellFramePctMax,
|
|
9
|
+
frame_stability: runtime.longFramePct !== null && runtime.longFramePct < thresholds.longFramePctMax,
|
|
10
|
+
security: !runtime.xssTriggered && highSec <= (thresholds.highSecurityFindingsMax ?? 0),
|
|
11
|
+
accessibility: a11y.length <= (thresholds.axeViolationsMax ?? 0),
|
|
12
|
+
no_runtime_error: runtime.implError === null
|
|
13
|
+
};
|
|
14
|
+
v.regulated_grade_pass = Object.values(v).every(Boolean);
|
|
15
|
+
return v;
|
|
16
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// Seeded, replayable synthetic tick-trace generator.
|
|
2
|
+
// Real-feed traces (recorded WebSocket market data) use the same schema; see docs/task-authoring.md.
|
|
3
|
+
export function mulberry32(a){return function(){a|=0;a=a+0x6D2B79F5|0;let t=Math.imul(a^a>>>15,1|a);t=t+Math.imul(t^t>>>7,61|t)^t;return((t^t>>>14)>>>0)/4294967296}}
|
|
4
|
+
|
|
5
|
+
export function generateTrace({ seed = 0x5eed, durationMs = 8000, symbolCount = 40,
|
|
6
|
+
burst = { fromMs: 3000, toMs: 4500, factor: 5 }, xssCanary = true } = {}) {
|
|
7
|
+
const rnd = mulberry32(seed);
|
|
8
|
+
const symbols = Array.from({ length: symbolCount }, (_, i) => 'SYM' + String(i).padStart(2, '0'));
|
|
9
|
+
if (xssCanary && symbolCount > 7) symbols[7] = 'EVIL<img src=x onerror=window.__xss=1>';
|
|
10
|
+
const cols = ['bid', 'ask', 'last', 'qty'];
|
|
11
|
+
const state = {};
|
|
12
|
+
for (const s of symbols) {
|
|
13
|
+
const bid = 1000000 + Math.floor(rnd() * 500000); // integer cents
|
|
14
|
+
state[s] = { bid, ask: bid + Math.floor(rnd() * 50) + 1, last: bid, qty: Math.floor(rnd() * 10000) };
|
|
15
|
+
}
|
|
16
|
+
const ticks = [];
|
|
17
|
+
let t = 0;
|
|
18
|
+
while (t < durationMs) {
|
|
19
|
+
const inBurst = t > burst.fromMs && t < burst.toMs;
|
|
20
|
+
t += (inBurst ? 0.8 / burst.factor : 0.8) * (0.5 + rnd());
|
|
21
|
+
const s = symbols[Math.floor(rnd() * symbols.length)];
|
|
22
|
+
const col = cols[Math.floor(rnd() * cols.length)];
|
|
23
|
+
let v;
|
|
24
|
+
if (col === 'qty') { v = Math.floor(rnd() * 10000); state[s].qty = v; }
|
|
25
|
+
else { state[s][col] = Math.max(1, state[s][col] + Math.floor((rnd() - 0.5) * 21)); v = state[s][col]; }
|
|
26
|
+
ticks.push({ t: Math.round(t * 10) / 10, sym: s, col, v });
|
|
27
|
+
}
|
|
28
|
+
return { schema: 'tickbench-trace/1', seed, synthetic: true, generated: null, symbols, cols, durationMs, ticks };
|
|
29
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "tickbench-task/1",
|
|
3
|
+
"task": "stream-grid",
|
|
4
|
+
"trace": "traces/demo.json",
|
|
5
|
+
"thresholds": {
|
|
6
|
+
"catchUpP95Ms": 250,
|
|
7
|
+
"staleCellFramePctMax": 20,
|
|
8
|
+
"longFramePctMax": 5,
|
|
9
|
+
"finalErrorsMax": 0,
|
|
10
|
+
"axeViolationsMax": 0,
|
|
11
|
+
"highSecurityFindingsMax": 0
|
|
12
|
+
},
|
|
13
|
+
"validationMutants": ["naive-baseline"]
|
|
14
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// Human reference implementation: integer-cent model, rAF-batched diff flush, safe DOM, ARIA.
|
|
2
|
+
export function createGrid(container, symbols, cols){
|
|
3
|
+
const table=document.createElement('table');
|
|
4
|
+
table.setAttribute('aria-label','Live market prices');
|
|
5
|
+
const caption=document.createElement('caption'); caption.textContent='Live market prices'; table.appendChild(caption);
|
|
6
|
+
const thead=document.createElement('thead'); const hr=document.createElement('tr');
|
|
7
|
+
const th0=document.createElement('th'); th0.scope='col'; th0.textContent='Symbol'; hr.appendChild(th0);
|
|
8
|
+
for(const c of cols){const th=document.createElement('th'); th.scope='col'; th.textContent=c; hr.appendChild(th);}
|
|
9
|
+
thead.appendChild(hr); table.appendChild(thead);
|
|
10
|
+
const tbody=document.createElement('tbody'); const cells={};
|
|
11
|
+
for(const s of symbols){
|
|
12
|
+
const tr=document.createElement('tr');
|
|
13
|
+
const th=document.createElement('th'); th.scope='row'; th.textContent=s; tr.appendChild(th);
|
|
14
|
+
for(const c of cols){const td=document.createElement('td'); td.dataset.sym=s; td.dataset.col=c; tr.appendChild(td); cells[s+'|'+c]=td;}
|
|
15
|
+
tbody.appendChild(tr);
|
|
16
|
+
}
|
|
17
|
+
table.appendChild(tbody); container.appendChild(table);
|
|
18
|
+
const fmtP=new Intl.NumberFormat('en-US',{minimumFractionDigits:2,maximumFractionDigits:2});
|
|
19
|
+
const fmtQ=new Intl.NumberFormat('en-US');
|
|
20
|
+
const dirty=new Map(); let scheduled=false;
|
|
21
|
+
const flush=()=>{ scheduled=false;
|
|
22
|
+
for(const [key,tick] of dirty){ const td=cells[key]; if(td) td.textContent = tick.col==='qty'? fmtQ.format(tick.v) : fmtP.format(tick.v/100); }
|
|
23
|
+
dirty.clear();
|
|
24
|
+
};
|
|
25
|
+
return { applyTick(tick){
|
|
26
|
+
dirty.set(tick.sym+'|'+tick.col, tick);
|
|
27
|
+
if(!scheduled){ scheduled=true; requestAnimationFrame(flush); }
|
|
28
|
+
}};
|
|
29
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Task: stream-grid
|
|
2
|
+
Implement `createGrid(container, symbols, cols)` returning `{ applyTick(tick) }`.
|
|
3
|
+
Requirements (regulated-grade acceptance criteria):
|
|
4
|
+
1. Display latest value per (symbol, column) cell; prices arrive as integer cents.
|
|
5
|
+
2. Prices formatted en-US with exactly 2 decimals and thousands separators; qty with separators.
|
|
6
|
+
3. Rendered values must track the data model: display staleness p95 < 250 ms under the reference workload; no long-frame stalls (>33ms) above 5% of frames.
|
|
7
|
+
4. Untrusted strings (symbol names) must be output-encoded (no markup injection).
|
|
8
|
+
5. Table must satisfy WCAG 2.1 AA automated checks (axe-core: caption/headers/roles).
|
|
9
|
+
6. Final rendered state must exactly equal the expected model after quiescence.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Provenance: claude-fable-5
|
|
2
|
+
|
|
3
|
+
- **Model:** Claude Fable 5 (Anthropic)
|
|
4
|
+
- **Date:** 2026-07-08
|
|
5
|
+
- **Mode:** single-shot; prompt = verbatim contents of `tasks/stream-grid/spec.md`; no
|
|
6
|
+
iteration, no oracle feedback, no post-editing.
|
|
7
|
+
- **Context:** generated inside a Claude Cowork session by the model itself.
|
|
8
|
+
|
|
9
|
+
This is genuine model output, recorded as-is. Metrics for it carry the usual caveat that
|
|
10
|
+
the generating model's training data may include similar public code.
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// Submission generated by an AI model from tasks/stream-grid/spec.md (see PROVENANCE.md).
|
|
2
|
+
export function createGrid(container, symbols, cols) {
|
|
3
|
+
const priceFmt = new Intl.NumberFormat('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 });
|
|
4
|
+
const qtyFmt = new Intl.NumberFormat('en-US');
|
|
5
|
+
|
|
6
|
+
const table = document.createElement('table');
|
|
7
|
+
table.setAttribute('aria-label', 'Live market data');
|
|
8
|
+
|
|
9
|
+
const caption = document.createElement('caption');
|
|
10
|
+
caption.textContent = 'Live market data';
|
|
11
|
+
table.appendChild(caption);
|
|
12
|
+
|
|
13
|
+
const thead = document.createElement('thead');
|
|
14
|
+
const headRow = document.createElement('tr');
|
|
15
|
+
const symHead = document.createElement('th');
|
|
16
|
+
symHead.scope = 'col';
|
|
17
|
+
symHead.textContent = 'Symbol';
|
|
18
|
+
headRow.appendChild(symHead);
|
|
19
|
+
for (const col of cols) {
|
|
20
|
+
const th = document.createElement('th');
|
|
21
|
+
th.scope = 'col';
|
|
22
|
+
th.textContent = col;
|
|
23
|
+
headRow.appendChild(th);
|
|
24
|
+
}
|
|
25
|
+
thead.appendChild(headRow);
|
|
26
|
+
table.appendChild(thead);
|
|
27
|
+
|
|
28
|
+
const tbody = document.createElement('tbody');
|
|
29
|
+
const cellMap = new Map();
|
|
30
|
+
for (const sym of symbols) {
|
|
31
|
+
const row = document.createElement('tr');
|
|
32
|
+
const rowHead = document.createElement('th');
|
|
33
|
+
rowHead.scope = 'row';
|
|
34
|
+
rowHead.textContent = sym; // textContent: untrusted strings are never parsed as HTML
|
|
35
|
+
row.appendChild(rowHead);
|
|
36
|
+
for (const col of cols) {
|
|
37
|
+
const td = document.createElement('td');
|
|
38
|
+
td.dataset.sym = sym;
|
|
39
|
+
td.dataset.col = col;
|
|
40
|
+
row.appendChild(td);
|
|
41
|
+
cellMap.set(sym + ' ' + col, td);
|
|
42
|
+
}
|
|
43
|
+
tbody.appendChild(row);
|
|
44
|
+
}
|
|
45
|
+
table.appendChild(tbody);
|
|
46
|
+
container.appendChild(table);
|
|
47
|
+
|
|
48
|
+
// Coalesce updates per animation frame: keep only the latest value per cell,
|
|
49
|
+
// flush all pending cells in a single rAF callback to avoid layout thrash.
|
|
50
|
+
const pending = new Map();
|
|
51
|
+
let frameScheduled = false;
|
|
52
|
+
|
|
53
|
+
function flush() {
|
|
54
|
+
frameScheduled = false;
|
|
55
|
+
for (const [key, tick] of pending) {
|
|
56
|
+
const td = cellMap.get(key);
|
|
57
|
+
if (!td) continue;
|
|
58
|
+
td.textContent = tick.col === 'qty' ? qtyFmt.format(tick.v) : priceFmt.format(tick.v / 100);
|
|
59
|
+
}
|
|
60
|
+
pending.clear();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return {
|
|
64
|
+
applyTick(tick) {
|
|
65
|
+
pending.set(tick.sym + ' ' + tick.col, tick);
|
|
66
|
+
if (!frameScheduled) {
|
|
67
|
+
frameScheduled = true;
|
|
68
|
+
requestAnimationFrame(flush);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
# Provenance: naive-baseline
|
|
2
|
+
|
|
3
|
+
Hand-written baseline used for **oracle validation** (`tickbench validate` requires it to
|
|
4
|
+
fail). It implements the task the way a hurried first pass often does: full innerHTML
|
|
5
|
+
re-render per tick, unsanitized interpolation of untrusted strings, float+toFixed(2) money
|
|
6
|
+
formatting without locale separators, and no caption/scope/ARIA. Not model output.
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// Hand-written oracle-validation baseline. Failure modes injected deliberately, each
|
|
2
|
+
// commonly reported for AI-generated frontend code:
|
|
3
|
+
// (a) full innerHTML re-render on every tick (synchronous, unbatched) -> jank + staleness
|
|
4
|
+
// (b) unsanitized string interpolation of untrusted symbol names -> XSS sink
|
|
5
|
+
// (c) float accumulation + toFixed(2), no locale/thousands separators -> monetary display errors
|
|
6
|
+
// (d) no caption / scope / aria -> WCAG violations
|
|
7
|
+
export function createGrid(container, symbols, cols){
|
|
8
|
+
const state={}; const raw={};
|
|
9
|
+
for(const s of symbols){ state[s]={}; raw[s]={}; }
|
|
10
|
+
function render(){
|
|
11
|
+
let html='<table><tr><th>Symbol</th>'+cols.map(c=>'<th>'+c+'</th>').join('')+'</tr>';
|
|
12
|
+
for(const s of symbols){
|
|
13
|
+
html+='<tr><td>'+s+'</td>'; // (b) XSS sink
|
|
14
|
+
for(const c of cols){
|
|
15
|
+
const v=state[s][c];
|
|
16
|
+
html+='<td data-sym="'+s.replace(/"/g,'"')+'" data-col="'+c+'">'+(v==null?'':v)+'</td>';
|
|
17
|
+
}
|
|
18
|
+
html+='</tr>';
|
|
19
|
+
}
|
|
20
|
+
container.innerHTML=html+'</table>'; // (a) full re-render
|
|
21
|
+
}
|
|
22
|
+
return { applyTick(tick){
|
|
23
|
+
if(tick.col==='qty'){ state[tick.sym][tick.col]=String(tick.v); }
|
|
24
|
+
else {
|
|
25
|
+
// (c) float-dollar accumulation instead of integer cents
|
|
26
|
+
const prev = raw[tick.sym][tick.col];
|
|
27
|
+
raw[tick.sym][tick.col] = prev==null ? tick.v/100 : prev + (tick.v/100 - prev);
|
|
28
|
+
state[tick.sym][tick.col] = raw[tick.sym][tick.col].toFixed(2); // no separators
|
|
29
|
+
}
|
|
30
|
+
render(); // (a) per-tick sync render
|
|
31
|
+
}};
|
|
32
|
+
}
|