personal-agent-benchmark 0.0.0-stage → 0.2.0

package/dist/src/report.js

@@ -0,0 +1,143 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.renderReport = renderReport;
+/** report.html: a self-contained page rendered from result.json. */
+function renderReport(r) {
+    const title = r.task.title ?? r.task.id;
+    const required = r.items.filter((i) => i.required);
+    const weighted = r.items.filter((i) => !i.required);
+    const score = r.score === undefined ? 'not graded' : `${Math.round(r.score * 1000) / 10}%`;
+    const summary = [
+        required.length ? `required ${required.filter((i) => i.passed).length}/${required.length}` : '',
+        weighted.length ? `weighted ${weighted.filter((i) => i.passed).length}/${weighted.length}` : '',
+        `${r.pages.not_found.length} page${r.pages.not_found.length === 1 ? '' : 's'} returned 404`,
+    ].filter(Boolean);
+    return `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>${esc(title)} - run report</title>
+<style>${CSS}</style>
+</head>
+<body>
+<main>
+<header>
+  <p class="eyebrow">${esc(r.task.id)}</p>
+  <h1>${esc(title)}</h1>
+  <p class="meta">graded ${esc(r.graded_at)}${r.grader ? ` by ${esc(r.grader.model)} (${esc(r.grader.via)})` : ''}${r.replay_log ? ` · replay log ${esc(r.replay_log)}` : ''}</p>
+</header>
+<section class="score ${r.score === undefined ? 'none' : r.score > 0 ? 'ok' : 'bad'}">
+  <div class="big">${esc(score)}</div>
+  <div>${summary.map(esc).join(' · ')}</div>
+</section>
+${r.notes.map((n) => `<p class="note">${esc(n)}</p>`).join('\n')}
+<section>
+  <h2>Pages that returned 404 <span class="count">${r.pages.not_found.length}</span></h2>
+  <p class="hint">Pages the agent opened in a tab during the replay (not frames, scripts or styles) that came back 404. "not recorded": the recording has no copy, so recording it would let the agent see it. "recorded as 404": the site itself answered 404 while recording.</p>
+  ${r.pages.not_found.length ? pageTable(r.pages.not_found, false) : '<p class="empty">None.</p>'}
+</section>
+${r.items.length ? `<section>
+  <h2>Items</h2>
+  <div class="table"><table>
+    <thead><tr><th>Result</th><th>Item</th><th>Test</th><th>Reason</th></tr></thead>
+    <tbody>${r.items.map(itemRow).join('')}</tbody>
+  </table></div>
+</section>` : ''}
+${Object.keys(r.fields).length ? `<section>
+  <h2>Extracted fields</h2>
+  <div class="table"><table>
+    <thead><tr><th>Field</th><th>Value</th></tr></thead>
+    <tbody>${Object.entries(r.fields).map(([k, v]) => `<tr><td><code>${esc(k)}</code></td><td>${v === null ? '<span class="muted">not stated</span>' : esc(JSON.stringify(v))}</td></tr>`).join('')}</tbody>
+  </table></div>
+</section>` : ''}
+<section>
+  <details>
+    <summary><h2>All pages opened <span class="count">${r.pages.opened.length}</span></h2></summary>
+    ${r.pages.opened.length ? pageTable(r.pages.opened, true) : '<p class="empty">None.</p>'}
+  </details>
+</section>
+${r.answer !== undefined ? `<section>
+  <h2>Agent answer</h2>
+  <pre class="answer">${esc(r.answer)}</pre>
+</section>` : ''}
+${r.task.prompt ? `<section>
+  <details>
+    <summary><h2>Task</h2></summary>
+    <pre class="answer">${esc(r.task.prompt)}</pre>
+  </details>
+</section>` : ''}
+</main>
+</body>
+</html>
+`;
+}
+function itemRow(i) {
+    const tag = i.required ? 'required' : `weight ${i.weight}`;
+    return `<tr class="${i.passed ? 'pass' : 'fail'}">
+    <td><span class="badge">${i.passed ? 'pass' : 'fail'}</span></td>
+    <td><code>${esc(i.id)}</code><div class="muted">${esc(i.kind)} · ${esc(tag)}</div></td>
+    <td>${esc(i.test)}</td>
+    <td>${esc(i.reason)}</td>
+  </tr>`;
+}
+const SOURCE = { miss: 'not recorded', blocked: 'blocked by a rule' };
+function pageTable(pages, all) {
+    const rows = pages.map((p) => {
+        const how = p.status === 404 ? (p.source === 'miss' ? 'not recorded' : 'recorded as 404') : SOURCE[p.source] ?? p.source;
+        return `<tr${p.status === 404 ? ' class="fail"' : ''}>
+      ${all ? `<td>${p.status}</td>` : ''}
+      <td>${esc(p.site)}</td>
+      <td><div class="route">${p.method === 'GET' ? '' : `${esc(p.method)} `}${esc(p.route)}</div><div class="url muted">${esc(p.url)}</div></td>
+      <td class="num">${p.count}</td>
+      <td class="nowrap">${esc(how)}</td>
+      <td class="muted nowrap">${esc(p.first.slice(11, 19))}</td>
+    </tr>`;
+    });
+    return `<div class="table"><table>
+    <thead><tr>${all ? '<th>Status</th>' : ''}<th>Site</th><th>Route</th><th>Times</th><th>${all ? 'Answered' : 'Why'}</th><th>First</th></tr></thead>
+    <tbody>${rows.join('')}</tbody>
+  </table></div>`;
+}
+function esc(s) {
+    return String(s).replace(/[&<>"']/g, (c) => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' })[c]);
+}
+const CSS = `
+:root { --bg: #fafaf9; --panel: #ffffff; --text: #1c1917; --muted: #78716c; --line: #e7e5e4; --ok: #15803d; --ok-bg: #f0fdf4; --bad: #b91c1c; --bad-bg: #fef2f2; --none-bg: #f5f5f4; --code: #f5f5f4; }
+@media (prefers-color-scheme: dark) { :root { --bg: #1c1917; --panel: #262220; --text: #f5f5f4; --muted: #a8a29e; --line: #3a3532; --ok: #4ade80; --ok-bg: #14261b; --bad: #f87171; --bad-bg: #2d1717; --none-bg: #292524; --code: #322d2a; } }
+* { box-sizing: border-box; }
+body { margin: 0; background: var(--bg); color: var(--text); font: 14px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; }
+main { max-width: 1120px; margin: 0 auto; padding: 32px 16px 64px; }
+h1 { font-size: 24px; margin: 4px 0 6px; }
+h2 { font-size: 17px; margin: 0 0 8px; display: inline-block; }
+section { margin-top: 28px; }
+.eyebrow { margin: 0; color: var(--muted); font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 12px; }
+.meta, .hint, .muted { color: var(--muted); }
+.meta { margin: 0; font-size: 12px; word-break: break-all; }
+.hint { margin: 0 0 10px; font-size: 13px; }
+.score { display: flex; gap: 20px; align-items: baseline; flex-wrap: wrap; padding: 16px 20px; border-radius: 10px; border: 1px solid var(--line); }
+.score.ok { background: var(--ok-bg); } .score.bad { background: var(--bad-bg); } .score.none { background: var(--none-bg); }
+.score .big { font-size: 32px; font-weight: 650; font-variant-numeric: tabular-nums; }
+.score.ok .big { color: var(--ok); } .score.bad .big { color: var(--bad); }
+.note { margin: 12px 0 0; padding: 10px 14px; border-left: 3px solid var(--bad); background: var(--panel); }
+.count { display: inline-block; min-width: 22px; padding: 0 7px; margin-left: 4px; border-radius: 11px; background: var(--code); color: var(--muted); font-size: 12px; text-align: center; vertical-align: 2px; }
+.table { overflow-x: auto; background: var(--panel); border: 1px solid var(--line); border-radius: 8px; }
+table { width: 100%; border-collapse: collapse; }
+td.nowrap, .nowrap { white-space: nowrap; }
+th, td { text-align: left; vertical-align: top; padding: 8px 10px; border-bottom: 1px solid var(--line); }
+th { font-size: 12px; font-weight: 600; color: var(--muted); white-space: nowrap; }
+tbody tr:last-child td { border-bottom: 0; }
+td.num { text-align: right; font-variant-numeric: tabular-nums; }
+.route { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 12.5px; word-break: break-all; }
+.url { font-size: 11.5px; word-break: break-all; margin-top: 2px; }
+code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 12.5px; background: var(--code); padding: 1px 5px; border-radius: 4px; }
+.badge { display: inline-block; padding: 1px 8px; border-radius: 10px; font-size: 12px; font-weight: 600; }
+tr.pass .badge { color: var(--ok); background: var(--ok-bg); }
+tr.fail .badge { color: var(--bad); background: var(--bad-bg); }
+tr.fail td:first-child { box-shadow: inset 3px 0 0 var(--bad); }
+.empty { color: var(--muted); margin: 0; }
+details > summary { cursor: pointer; list-style-position: outside; margin-left: 16px; }
+details[open] > summary { margin-bottom: 8px; }
+pre.answer { white-space: pre-wrap; word-break: break-word; background: var(--panel); border: 1px solid var(--line); border-radius: 8px; padding: 14px 16px; margin: 0; font: 13px/1.55 ui-monospace, SFMono-Regular, Menlo, monospace; }
+`;
+//# sourceMappingURL=report.js.map

package/dist/src/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/report.ts"],"names":[],"mappings":";;;AAEA,oEAAoE;AACpE,sBAA6B,CAAY;IACvC,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;IACxC,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC;IAC3F,MAAM,OAAO,GAAG;QACd,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE;QAC/F,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE;QAC/F,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,QAAQ,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,eAAe;KAC5F,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAClB,OAAO;;;;;SAKA,GAAG,CAAC,KAAK,CAAC;SACV,GAAG;;;;;uBAKW,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,GAAG,CAAC,KAAK,CAAC;2BACS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;;wBAEpJ,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK;qBAC9D,GAAG,CAAC,KAAK,CAAC;SACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;;EAEnC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mBAAmB,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;oDAEZ,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM;;IAExE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,4BAA4B;;EAE/F,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;;;;aAIN,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;;WAE/B,CAAC,CAAC,CAAC,EAAE;EACd,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;;;;aAIpB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,GAAG,CAAC,CAAC,CAAC,mBAAmB,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,uCAAuC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;;WAExL,CAAC,CAAC,CAAC,EAAE;;;wDAGwC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM;MACvE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,4BAA4B;;;EAG1F,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC;;wBAEH,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;WAC1B,CAAC,CAAC,CAAC,EAAE;EACd,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;;;0BAGQ,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;;WAEjC,CAAC,CAAC,CAAC,EAAE;;;;CAIf,CAAC;AACF,CAAC;AAED,SAAS,OAAO,CAAC,CAAa;IAC5B,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,CAAC;IAC3D,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;8BACnB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;gBACxC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,6BAA6B,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC;UACrE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;UACX,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QACf,CAAC;AACT,CAAC;AAED,MAAM,MAAM,GAA2B,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC;AAE9F,SAAS,SAAS,CAAC,KAAmB,EAAE,GAAY;IAClD,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QACzH,OAAO,MAAM,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE;QAChD,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,EAAE;YAC7B,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;+BACQ,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,gCAAgC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;wBAC7G,CAAC,CAAC,KAAK;2BACJ,GAAG,CAAC,GAAG,CAAC;iCACF,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;UACjD,CAAC;IACT,CAAC,CAAC,CAAC;IACH,OAAO;iBACQ,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,gDAAgD,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK;aACxG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;iBACT,CAAC;AAClB,CAAC;AAED,SAAS,GAAG,CAAC,CAAU;IACrB,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;AAC7H,CAAC;AAED,MAAM,GAAG,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqCX,CAAC"}

package/dist/src/scoring.d.ts

@@ -0,0 +1,87 @@
+import type { RuleSet } from './core/rules';
+import type { Grading } from './grading';
+import type { Judge } from './judge';
+/**
+ * Scores one agent run (docs/grading.md) and lists the pages the agent opened in the replay, so
+ * pages that came back 404 can be added to the recording.
+ */
+/** A line of the replay request log (ReplayEngine.record). */
+export interface LogLine {
+    t: string;
+    method: string;
+    url: string;
+    key: string;
+    class: string;
+    /** sec-fetch-dest; absent in logs written before it was recorded and for non-browser clients. */
+    dest?: string;
+    source: string;
+    status: number;
+}
+export interface ItemResult {
+    kind: 'require' | 'check' | 'trace';
+    id: string;
+    /** What was tested: the question, the expected value or the request. */
+    test: string;
+    required: boolean;
+    weight?: number;
+    passed: boolean;
+    reason: string;
+}
+/** A page (one normalized key) opened in a browser tab during the replay. */
+export interface PageResult {
+    site: string;
+    /** The normalized key without its method: host, path and the query parameters that matter. */
+    route: string;
+    method: string;
+    /** The first URL the agent opened for it. */
+    url: string;
+    count: number;
+    status: number;
+    /** How replay answered: exact, fuzzy, search, promo, blocked, or miss (not in the recording). */
+    source: string;
+    first: string;
+}
+export interface RunResult {
+    task: {
+        id: string;
+        title?: string;
+        prompt?: string;
+    };
+    graded_at: string;
+    replay_log?: string;
+    answer?: string;
+    grader?: {
+        via: string;
+        model: string;
+    };
+    /** 0 to 1; absent when the run could not be graded (see notes). */
+    score?: number;
+    fields: Record<string, unknown>;
+    items: ItemResult[];
+    /** Pages in browser tabs, not frames, scripts or styles. not_found: those answered with 404. */
+    pages: {
+        opened: PageResult[];
+        not_found: PageResult[];
+    };
+    notes: string[];
+}
+export interface RunInput {
+    task: RunResult['task'];
+    grading?: Grading;
+    answer?: string;
+    log: LogLine[];
+    logFile?: string;
+    rules: RuleSet;
+    /** Each site's recorded 404 page (manifest not_found): recorder probes, not pages anyone opened. */
+    probes?: string[];
+    judge?: Judge;
+}
+export declare function scoreRun(input: RunInput): Promise<RunResult>;
+/** 0 when a required item failed; otherwise the passed share of the weighted checks, or 1. */
+export declare function score(items: ItemResult[]): number;
+/** Writes result.json and report.html (rendered from it) into `dir`. */
+export declare function writeRun(result: RunResult, dir: string): {
+    json: string;
+    html: string;
+};
+export declare function isPage(r: LogLine): boolean;

package/dist/src/scoring.js

@@ -0,0 +1,232 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scoreRun = scoreRun;
+exports.score = score;
+exports.writeRun = writeRun;
+exports.isPage = isPage;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const util_1 = require("./core/util");
+const report_1 = require("./report");
+async function scoreRun(input) {
+    const { grading, answer } = input;
+    const result = {
+        task: input.task,
+        graded_at: new Date().toISOString(),
+        replay_log: input.logFile,
+        answer,
+        fields: {},
+        items: [],
+        pages: pagesOf(input.log, input.rules, new Set(input.probes ?? [])),
+        notes: [],
+    };
+    if (!grading) {
+        result.notes.push('not graded: the task has no grading.yaml');
+        return result;
+    }
+    if (answer === undefined) {
+        result.notes.push('not graded: no answer was given (--answer)');
+        return result;
+    }
+    const extract = grading.extract ?? {};
+    const checks = grading.checks ?? [];
+    let reply = { fields: {}, checks: {} };
+    if (Object.keys(extract).length || checks.length) {
+        if (!input.judge)
+            throw new Error('grading needs a judge');
+        result.grader = { via: input.judge.via, model: input.judge.model };
+        try {
+            const asked = await input.judge.ask(judgeRequest(input.task, grading, answer));
+            reply = asked.output;
+            result.grader.model = asked.model;
+        }
+        catch (err) {
+            result.notes.push(`not graded: the grader failed (${err.message})`);
+            return result;
+        }
+    }
+    result.fields = reply.fields ?? {};
+    for (const r of grading.require ?? []) {
+        const type = extract[r.field]?.type ?? 'text';
+        result.items.push({ kind: 'require', id: r.field, test: describe(r), required: true, ...compare(r, type, result.fields[r.field]) });
+    }
+    for (const c of checks) {
+        const got = reply.checks?.[c.id];
+        const weighted = c.weight !== undefined;
+        result.items.push({
+            kind: 'check',
+            id: c.id,
+            test: c.question,
+            required: !weighted,
+            ...(weighted ? { weight: c.weight } : {}),
+            passed: got?.pass === true,
+            reason: got?.reason ?? 'the grader gave no answer',
+        });
+    }
+    for (const t of grading.trace ?? []) {
+        const rule = (t.must ?? t.must_not);
+        const found = input.log.filter((r) => matches(r, rule));
+        const first = found[0] && `${found[0].method} ${found[0].url}${found.length > 1 ? ` (and ${found.length - 1} more)` : ''}`;
+        const passed = t.must ? found.length > 0 : found.length === 0;
+        result.items.push({ kind: 'trace', id: t.id, test: `${t.must ? 'must' : 'must not'} request ${matchText(rule)}`, required: true, passed, reason: first ?? 'no such request' });
+    }
+    result.score = score(result.items);
+    return result;
+}
+/** 0 when a required item failed; otherwise the passed share of the weighted checks, or 1. */
+function score(items) {
+    if (items.some((i) => i.required && !i.passed))
+        return 0;
+    const weighted = items.filter((i) => i.weight !== undefined);
+    const total = weighted.reduce((s, i) => s + i.weight, 0);
+    return total ? weighted.filter((i) => i.passed).reduce((s, i) => s + i.weight, 0) / total : 1;
+}
+/** Writes result.json and report.html (rendered from it) into `dir`. */
+function writeRun(result, dir) {
+    fs.mkdirSync(dir, { recursive: true });
+    const json = path.join(dir, 'result.json');
+    const html = path.join(dir, 'report.html');
+    fs.writeFileSync(json, JSON.stringify(result, null, 2) + '\n');
+    fs.writeFileSync(html, (0, report_1.renderReport)(result));
+    return { json, html };
+}
+// ---- pages
+const ASSET = /\.(m?js|css|map|json|xml|txt|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|mp4|webm|mp3|wasm)$/i;
+const NOT_REPLAYED = new Set(['control', 'preflight']);
+function isPage(r) {
+    if (NOT_REPLAYED.has(r.source))
+        return false;
+    // a request without sec-fetch-dest (older logs, non-browser clients) counts when replay classed it as a page
+    if (r.dest !== undefined ? r.dest !== 'document' : r.class !== 'document')
+        return false;
+    try {
+        return !ASSET.test(new URL(r.url).pathname);
+    }
+    catch {
+        return false;
+    }
+}
+function pagesOf(log, rules, probes) {
+    const byKey = new Map();
+    for (const r of log) {
+        if (!isPage(r) || probes.has(r.url))
+            continue;
+        const id = `${r.status} ${r.key}`;
+        const seen = byKey.get(id);
+        if (seen) {
+            seen.count++;
+            continue;
+        }
+        const host = new URL(r.url).host;
+        const route = r.key ? r.key.slice(r.key.indexOf(' ') + 1) : host + new URL(r.url).pathname;
+        byKey.set(id, { site: rules.siteFor(host)?.site ?? host, route, method: r.method, url: r.url, count: 1, status: r.status, source: r.source, first: r.t });
+    }
+    const opened = [...byKey.values()];
+    return { opened, not_found: opened.filter((p) => p.status === 404) };
+}
+const SYSTEM = [
+    'You grade the answer a personal AI agent gave to a task it was set.',
+    'Extract the listed fields from the answer and answer each yes/no question about the answer.',
+    'Take values only from what the answer says; when the answer does not state a field, return null for it.',
+    'For each question give a short reason first, quoting or pointing to the part of the answer it rests on, then pass: true for yes and false for no.',
+].join(' ');
+const JSON_TYPE = { money: 'number', number: 'number', text: 'string', boolean: 'boolean' };
+function judgeRequest(task, g, answer) {
+    const extract = Object.entries(g.extract ?? {});
+    const checks = g.checks ?? [];
+    const properties = {};
+    if (extract.length) {
+        properties.fields = {
+            type: 'object',
+            properties: Object.fromEntries(extract.map(([name, f]) => [name, { anyOf: [{ type: JSON_TYPE[f.type] }, { type: 'null' }], ...(f.description ? { description: f.description } : {}) }])),
+            required: extract.map(([name]) => name),
+            additionalProperties: false,
+        };
+    }
+    if (checks.length) {
+        const verdict = { type: 'object', properties: { reason: { type: 'string' }, pass: { type: 'boolean' } }, required: ['reason', 'pass'], additionalProperties: false };
+        properties.checks = { type: 'object', properties: Object.fromEntries(checks.map((c) => [c.id, verdict])), required: checks.map((c) => c.id), additionalProperties: false };
+    }
+    const lines = [`<task>\n${task.prompt ?? task.title ?? task.id}\n</task>`, `<answer>\n${answer}\n</answer>`];
+    if (extract.length)
+        lines.push('Fields to extract (money and number as plain numbers, without currency signs):', ...extract.map(([name, f]) => `- ${name} (${f.type})${f.description ? `: ${f.description}` : ''}`));
+    if (checks.length)
+        lines.push('Questions:', ...checks.map((c) => `- ${c.id}: ${c.question}`));
+    return { system: SYSTEM, prompt: lines.join('\n\n'), schema: { type: 'object', properties, required: Object.keys(properties), additionalProperties: false } };
+}
+// ---- program-checked items
+const squash = (s) => String(s).replace(/\s+/g, ' ').trim().toLowerCase();
+function describe(r) {
+    const parts = [];
+    if (r.equals !== undefined)
+        parts.push(`equals ${r.equals}${r.tolerance !== undefined ? ` (±${r.tolerance})` : ''}`);
+    if (r.min !== undefined)
+        parts.push(`at least ${r.min}`);
+    if (r.max !== undefined)
+        parts.push(`at most ${r.max}`);
+    if (r.one_of !== undefined)
+        parts.push(`one of ${r.one_of.join(', ')}`);
+    if (r.contains !== undefined)
+        parts.push(`contains "${r.contains}"`);
+    return `${r.field} ${parts.join(', ')}`;
+}
+function compare(r, type, value) {
+    if (value === null || value === undefined)
+        return { passed: false, reason: 'not stated in the answer' };
+    const got = `got ${JSON.stringify(value)}`;
+    if (type === 'money' || type === 'number') {
+        const v = Number(value);
+        const tol = r.tolerance ?? 0.01;
+        const near = (x) => Math.abs(v - Number(x)) <= tol + 1e-9;
+        const ok = !Number.isNaN(v) && (r.equals === undefined || near(r.equals)) && (r.min === undefined || v >= r.min) && (r.max === undefined || v <= r.max) && (r.one_of === undefined || r.one_of.some(near));
+        return { passed: ok, reason: got };
+    }
+    if (type === 'boolean')
+        return { passed: value === r.equals, reason: got };
+    const v = squash(value);
+    const ok = (r.equals === undefined || v === squash(r.equals)) && (r.one_of === undefined || r.one_of.some((o) => v === squash(o))) && (r.contains === undefined || v.includes(squash(r.contains)));
+    return { passed: ok, reason: got };
+}
+const NOT_FROM_RECORDING = new Set(['miss', 'blocked', 'preflight', 'control']);
+function matches(r, m) {
+    if (m.method && r.method.toUpperCase() !== m.method.toUpperCase())
+        return false;
+    if (m.hit && NOT_FROM_RECORDING.has(r.source))
+        return false;
+    return (0, util_1.globToRegExp)(m.url).test(r.url);
+}
+const matchText = (m) => `${m.method ? `${m.method.toUpperCase()} ` : ''}${m.url}${m.hit ? ' (answered from the recording)' : ''}`;
+//# sourceMappingURL=scoring.js.map

package/dist/src/scoring.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scoring.js","sourceRoot":"","sources":["../../src/scoring.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,MAAY,EAAE,oCAAgB;AAC9B,MAAY,IAAI,sCAAkB;AAElC,sCAA2C;AAG3C,qCAAwC;AAyEjC,KAAK,mBAAmB,KAAe;IAC5C,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;IAClC,MAAM,MAAM,GAAc;QACxB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,UAAU,EAAE,KAAK,CAAC,OAAO;QACzB,MAAM;QACN,MAAM,EAAE,EAAE;QACV,KAAK,EAAE,EAAE;QACT,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QACnE,KAAK,EAAE,EAAE;KACV,CAAC;IACF,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;QAC9D,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;QAChE,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;IACtC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;IACpC,IAAI,KAAK,GAAe,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACnD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACjD,IAAI,CAAC,KAAK,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnE,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;YAC/E,KAAK,GAAG,KAAK,CAAC,MAAoB,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QACpC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,kCAAmC,GAAa,CAAC,OAAO,GAAG,CAAC,CAAC;YAC/E,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IACD,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC;IAEnC,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,MAAM,CAAC;QAC9C,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;IACtI,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,OAAO;YACb,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,QAAQ;YAChB,QAAQ,EAAE,CAAC,QAAQ;YACnB,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzC,MAAM,EAAE,GAAG,EAAE,IAAI,KAAK,IAAI;YAC1B,MAAM,EAAE,GAAG,EAAE,MAAM,IAAI,2BAA2B;SACnD,CAAC,CAAC;IACL,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,QAAQ,CAAE,CAAC;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;QACxD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,MAAM,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC3H,MAAM,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;QAC9D,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,YAAY,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI,iBAAiB,EAAE,CAAC,CAAC;IACjL,CAAC;IACD,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8FAA8F;AAC9F,eAAsB,KAAmB;IACvC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAO,EAAE,CAAC,CAAC,CAAC;IAC1D,OAAO,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAO,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACjG,CAAC;AAED,wEAAwE;AACxE,kBAAyB,MAAiB,EAAE,GAAW;IACrD,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;IAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;IAC3C,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAC/D,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,IAAA,qBAAY,EAAC,MAAM,CAAC,CAAC,CAAC;IAC7C,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AACxB,CAAC;AAED,aAAa;AAEb,MAAM,KAAK,GAAG,sGAAsG,CAAC;AACrH,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC;AAEvD,gBAAuB,CAAU;IAC/B,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7C,6GAA6G;IAC7G,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,UAAU;QAAE,OAAO,KAAK,CAAC;IACxF,IAAI,CAAC;QACH,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,OAAO,CAAC,GAAc,EAAE,KAAc,EAAE,MAAmB;IAClE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC5C,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAAE,SAAS;QAC9C,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3B,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;QACjC,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAC3F,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC5J,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACnC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,EAAE,CAAC;AACvE,CAAC;AASD,MAAM,MAAM,GAAG;IACb,qEAAqE;IACrE,6FAA6F;IAC7F,yGAAyG;IACzG,mJAAmJ;CACpJ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEZ,MAAM,SAAS,GAA8B,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAEvH,SAAS,YAAY,CAAC,IAAuB,EAAE,CAAU,EAAE,MAAc;IACvE,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;IAC9B,MAAM,UAAU,GAA4B,EAAE,CAAC;IAC/C,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,UAAU,CAAC,MAAM,GAAG;YAClB,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACxL,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC;YACvC,oBAAoB,EAAE,KAAK;SAC5B,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;QACrK,UAAU,CAAC,MAAM,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;IAC7K,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,WAAW,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,WAAW,EAAE,aAAa,MAAM,aAAa,CAAC,CAAC;IAC7G,IAAI,OAAO,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,gFAAgF,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACrN,IAAI,MAAM,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAC9F,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,oBAAoB,EAAE,KAAK,EAAE,EAAE,CAAC;AAChK,CAAC;AAED,6BAA6B;AAE7B,MAAM,MAAM,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAEnF,SAAS,QAAQ,CAAC,CAAc;IAC9B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACrH,IAAI,CAAC,CAAC,GAAG,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;IACzD,IAAI,CAAC,CAAC,GAAG,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;IACxD,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxE,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;IACrE,OAAO,GAAG,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,OAAO,CAAC,CAAc,EAAE,IAAe,EAAE,KAAc;IAC9D,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACxG,MAAM,GAAG,GAAG,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;IAC3C,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QACxB,MAAM,GAAG,GAAG,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC;QAChC,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC;QACnE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3M,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IACrC,CAAC;IACD,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,EAAE,MAAM,EAAE,KAAK,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IAC3E,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IACxB,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACnM,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;AACrC,CAAC;AAED,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC;AAEhF,SAAS,OAAO,CAAC,CAAU,EAAE,CAAe;IAC1C,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE;QAAE,OAAO,KAAK,CAAC;IAChF,IAAI,CAAC,CAAC,GAAG,IAAI,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5D,OAAO,IAAA,mBAAY,EAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,SAAS,GAAG,CAAC,CAAe,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,gCAAgC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC"}

package/dist/src/tasks.d.ts

@@ -0,0 +1,61 @@
+/**
+ * tasks/<id>/ in this repository:
+ *   task.yaml        the task card (what the agent is asked)
+ *   recording/       the recording, in the published layout (core/archive.ts, docs/bundle-format.md)
+ *   rules/           site rules for this task: <site>.auto.yaml inferred from the recording,
+ *                    <site>.yaml written by people (wins over auto and over sites/)
+ *   grading.yaml     how answers to the task are scored (docs/grading.md)
+ */
+export interface TaskCard {
+    id: string;
+    title?: string;
+    /** The instruction given to the agent. */
+    prompt?: string;
+    /** The browser's language and time zone while recording; replay keeps them. */
+    capture?: {
+        locale?: string;
+        timezone?: string;
+    };
+}
+/** Everything a replay needs: the recording and the rules layered over it. */
+export interface ReplayTarget {
+    name: string;
+    bundleDir: string;
+    /** Rule directories layered over the bundle's own rules/ (later wins). */
+    rulesDirs: string[];
+    task?: Task;
+}
+export declare class Repo {
+    readonly root: string;
+    constructor(root: string);
+    /** $PAB_REPO, or the checkout this code runs from. */
+    static default(): Repo;
+    get sitesDir(): string;
+    get tasksDir(): string;
+    /** Unpacked bundles and replay logs ($PAB_CACHE, default .cache/ in the repository). */
+    get cacheDir(): string;
+    taskIds(): string[];
+    task(id: string): Task;
+}
+export declare class Task {
+    readonly repo: Repo;
+    readonly dir: string;
+    constructor(repo: Repo, dir: string);
+    get id(): string;
+    get cardPath(): string;
+    get recordingDir(): string;
+    get rulesDir(): string;
+    /** grading.yaml: how answers are scored (see grading.ts). */
+    get gradingPath(): string;
+    card(): TaskCard;
+    /** Files of the recording; empty when the task has none yet. */
+    recordingFiles(): string[];
+    /** Unpacks the recording into the cache (once per recording content) and returns its directory. */
+    bundleDir(): Promise<string>;
+    target(): Promise<ReplayTarget>;
+}
+/**
+ * Resolves a command-line argument to a replay target: a task id, a task directory, or a bundle
+ * directory (one with manifest.json, replayed with its own rules/).
+ */
+export declare function resolveTarget(arg: string, repo?: Repo): Promise<ReplayTarget>;