personal-agent-benchmark 0.0.0-stage → 0.2.0

package/dist/src/core/writer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"writer.js","sourceRoot":"","sources":["../../../src/core/writer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,MAAY,EAAE,oCAAgB;AAC9B,MAAY,IAAI,sCAAkB;AAIlC,iCAAuE;AACvE,iCAAoC;AAsBpC,mGAAmG;AACnG,MAAM,YAAY,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;AAE9E;;;;GAIG;AACI,KAAK,sBAAsB,MAAc,EAAE,SAAqB,EAAE,CAAqB;IAC5F,IAAI,CAAC,SAAS,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;IAChE,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,GAAG,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAE7E,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9E,KAAK,MAAM,CAAC,IAAI,YAAY;QAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnF,MAAM,MAAM,GAAG,IAAI,iBAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC;IAC7E,MAAM,aAAa,GAAG,IAAI,GAAG,EAAoB,CAAC;IAClD,MAAM,SAAS,GAAG,CAAC,CAAC,cAAc,IAAI,IAAI,CAAC;IAC3C,MAAM,OAAO,GAAY,EAAE,CAAC;IAC5B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAA,aAAM,EAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACtF,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC7G,IAAI,KAAK;YAAE,QAAQ,EAAE,CAAC;aACjB,CAAC;YACJ,KAAK,IAAI,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC5B,IAAI,EAAE,CAAC,QAAQ,CAAC,MAAM,IAAI,SAAS;gBAAE,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC;YACX,EAAE,EAAE,EAAE,CAAC,EAAE;YACT,EAAE,EAAE,EAAE,CAAC,EAAE;YACT,MAAM,EAAE,EAAE,CAAC,MAAM;YACjB,GAAG,EAAE,EAAE,CAAC,GAAG;YACX,MAAM,EAAE,EAAE,CAAC,MAAM;YACjB,IAAI,EAAE,IAAA,aAAM,EAAC,EAAE,CAAC,WAAW,CAAC;YAC5B,YAAY,EAAE,EAAE,CAAC,YAAY;YAC7B,cAAc,EAAE,IAAA,aAAM,EAAC,EAAE,CAAC,UAAU,EAAE,cAAc,CAAC;YACrD,OAAO,EAAE,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS;YACpF,MAAM;YACN,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACxB,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,MAAM;YAC3B,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,MAAM;YAC3B,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM;YAC7B,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM;YAC7B,SAAS,EAAE,KAAK,EAAE,EAAE;SACrB,CAAC,CAAC;IACL,CAAC;IACD,MAAM,CAAC,KAAK,EAAE,CAAC;IACf,IAAA,iBAAU,EAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IAExC,MAAM,KAAK,GAAiB,CAAC,CAAC,KAAK,EAAE,MAAM;QACzC,CAAC,CAAC,CAAC,CAAC,KAAK;QACT,CAAC,CAAC,OAAO;aACJ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,CAAC,YAAY,IAAI,UAAU,CAAC,KAAK,UAAU,CAAC;aAChI,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACrF,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACvF,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,CAAC,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IACvG,IAAA,iBAAU,EAAC,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAEpC,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,iBAAiB,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QACrC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC;IAC1D,CAAC;IACD,MAAM,QAAQ,GAAa;QACzB,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO;QACvB,aAAa,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE;QAC5B,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE;QAC3C,SAAS,EAAE,QAAQ;QACnB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM;QACrB,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ;KAC1B,CAAC;IACF,IAAA,gBAAS,EAAC,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACzC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AAChD,CAAC"}

package/dist/src/grading.d.ts

@@ -0,0 +1,63 @@
+import type { RecordStore } from './core/store';
+/**
+ * tasks/<id>/grading.yaml: how an agent's answer to the task is scored (docs/grading.md).
+ * An LLM extracts fields from the answer and answers yes/no questions; the program compares the
+ * extracted fields with the expected ones and checks the requests the agent made in the replay.
+ */
+export type FieldType = 'money' | 'number' | 'text' | 'boolean';
+export interface Grading {
+    /** Fields an LLM extracts from the agent's answer. */
+    extract?: Record<string, {
+        type: FieldType;
+        description?: string;
+    }>;
+    /** Compared by the program; any failure scores 0. */
+    require?: Requirement[];
+    /** Yes/no questions an LLM answers about the answer. */
+    checks?: Check[];
+    /** Requests the agent must or must not have made, from the replay log. */
+    trace?: TraceRule[];
+}
+export interface Requirement {
+    field: string;
+    equals?: number | string | boolean;
+    min?: number;
+    max?: number;
+    one_of?: (string | number)[];
+    contains?: string;
+    /** For numbers: how far the answer may be from equals (default 0.01). */
+    tolerance?: number;
+    /** Where the expected value appears in the recording: a page and its text as shown there. */
+    source?: {
+        url: string;
+        text: string;
+    };
+}
+export interface Check {
+    id: string;
+    question: string;
+    /** A "no" scores 0 (the default when no weight is given). */
+    required?: boolean;
+    /** Partial credit instead: counts toward the score once every required item passed. */
+    weight?: number;
+}
+export interface TraceRule {
+    id: string;
+    must?: RequestMatch;
+    must_not?: RequestMatch;
+}
+export interface RequestMatch {
+    method?: string;
+    /** Glob over the full URL; * matches anything. */
+    url: string;
+    /** Only requests the replay answered from the recording (not misses). */
+    hit?: boolean;
+}
+export declare function parseGrading(text: string): Grading;
+/** What is wrong with a grading, as sentences; empty when it can be used. */
+export declare function validateGrading(g: Grading): string[];
+/**
+ * Expected values whose source page is not in the recording, or does not show the text: the
+ * answer could not have been read during replay.
+ */
+export declare function checkSources(g: Grading, store: RecordStore): Promise<string[]>;

package/dist/src/grading.js

@@ -0,0 +1,173 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseGrading = parseGrading;
+exports.validateGrading = validateGrading;
+exports.checkSources = checkSources;
+const cheerio = __importStar(require("cheerio"));
+const yaml_1 = __importDefault(require("yaml"));
+const lookup_1 = require("./core/lookup");
+const normalize_1 = require("./core/normalize");
+const TYPES = ['money', 'number', 'text', 'boolean'];
+const KEYS = ['extract', 'require', 'checks', 'trace'];
+const ID = /^[a-z][a-z0-9_]*$/;
+function parseGrading(text) {
+    const value = yaml_1.default.parse(text);
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        throw new Error('grading.yaml must be a mapping');
+    return value;
+}
+/** What is wrong with a grading, as sentences; empty when it can be used. */
+function validateGrading(g) {
+    const errors = [];
+    for (const k of Object.keys(g))
+        if (!KEYS.includes(k))
+            errors.push(`unknown key "${k}" (expected ${KEYS.join(', ')})`);
+    const fields = g.extract ?? {};
+    if (typeof fields !== 'object' || Array.isArray(fields))
+        errors.push('extract must map field names to { type, description }');
+    for (const [name, f] of Object.entries(fields)) {
+        if (!ID.test(name))
+            errors.push(`extract: field name "${name}" must be lowercase letters, digits and _`);
+        if (!f || !TYPES.includes(f.type))
+            errors.push(`extract.${name}: type must be one of ${TYPES.join(', ')}`);
+        if (f?.description !== undefined && typeof f.description !== 'string')
+            errors.push(`extract.${name}: description must be text (quote it if it has a colon)`);
+    }
+    const list = (key, v) => {
+        if (v !== undefined && !Array.isArray(v))
+            errors.push(`${key} must be a list`);
+        return Array.isArray(v) ? v : [];
+    };
+    const require = list('require', g.require);
+    const checks = list('checks', g.checks);
+    const trace = list('trace', g.trace);
+    if (!require.length && !checks.length && !trace.length)
+        errors.push('nothing to grade: add require, checks or trace');
+    require.forEach((r, i) => {
+        const at = `require[${i}]`;
+        const f = fields[r?.field];
+        if (!f)
+            return errors.push(`${at}: field "${r?.field}" is not in extract`);
+        const tests = ['equals', 'min', 'max', 'one_of', 'contains'].filter((k) => r[k] !== undefined);
+        if (!tests.length)
+            errors.push(`${at}: give equals, min/max, one_of or contains`);
+        if (r.equals !== undefined && (r.min !== undefined || r.max !== undefined || r.one_of !== undefined || r.contains !== undefined))
+            errors.push(`${at}: equals cannot be combined with other tests`);
+        const numeric = f.type === 'money' || f.type === 'number';
+        if (numeric) {
+            for (const k of ['equals', 'min', 'max'])
+                if (r[k] !== undefined && typeof r[k] !== 'number')
+                    errors.push(`${at}: ${k} must be a number for a ${f.type} field`);
+            if (r.contains !== undefined)
+                errors.push(`${at}: contains is for text fields`);
+            if (r.tolerance !== undefined && !(typeof r.tolerance === 'number' && r.tolerance >= 0))
+                errors.push(`${at}: tolerance must be a number of 0 or more`);
+        }
+        else {
+            if (r.min !== undefined || r.max !== undefined || r.tolerance !== undefined)
+                errors.push(`${at}: min, max and tolerance are for number and money fields`);
+            if (f.type === 'boolean' && (typeof r.equals !== 'boolean' || tests.length > 1))
+                errors.push(`${at}: a boolean field takes equals: true or false`);
+            if (f.type === 'text' && r.equals !== undefined && typeof r.equals !== 'string')
+                errors.push(`${at}: equals must be text for a text field`);
+        }
+        if (r.one_of !== undefined && (!Array.isArray(r.one_of) || !r.one_of.length))
+            errors.push(`${at}: one_of must be a non-empty list`);
+        if (r.source !== undefined && !(typeof r.source?.url === 'string' && /^https?:\/\//.test(r.source.url) && typeof r.source.text === 'string' && r.source.text.trim()))
+            errors.push(`${at}: source needs an http(s) url and the text shown there`);
+    });
+    const ids = new Set();
+    const id = (at, v) => {
+        if (typeof v !== 'string' || !ID.test(v))
+            errors.push(`${at}: id must be lowercase letters, digits and _`);
+        else if (ids.has(v))
+            errors.push(`${at}: id "${v}" is used twice`);
+        else
+            ids.add(v);
+    };
+    checks.forEach((c, i) => {
+        const at = `checks[${i}]`;
+        id(at, c?.id);
+        if (typeof c?.question !== 'string' || !c.question.trim())
+            errors.push(`${at}: question is missing`);
+        if (c?.weight !== undefined && !(typeof c.weight === 'number' && c.weight > 0))
+            errors.push(`${at}: weight must be a positive number`);
+        if (c?.weight !== undefined && c.required)
+            errors.push(`${at}: a check is either required or weighted`);
+    });
+    trace.forEach((t, i) => {
+        const at = `trace[${i}]`;
+        id(at, t?.id);
+        const rules = [t?.must, t?.must_not].filter(Boolean);
+        if (rules.length !== 1)
+            errors.push(`${at}: give exactly one of must or must_not`);
+        for (const m of rules)
+            if (typeof m?.url !== 'string' || !m.url)
+                errors.push(`${at}: url is missing`);
+    });
+    return errors;
+}
+/**
+ * Expected values whose source page is not in the recording, or does not show the text: the
+ * answer could not have been read during replay.
+ */
+async function checkSources(g, store) {
+    const errors = [];
+    for (const [i, r] of (g.require ?? []).entries()) {
+        if (!r.source)
+            continue;
+        const norm = (0, normalize_1.normalizeRequest)({ method: 'GET', url: r.source.url }, store.rules);
+        // the copy replay would answer with (the last one for carts and checkouts)
+        const ix = (0, lookup_1.lookup)(store, norm)?.ix ?? store.findByUrl(r.source.url);
+        if (!ix) {
+            errors.push(`require[${i}] (${r.field}): ${r.source.url} is not in the recording`);
+            continue;
+        }
+        // the text a page shows, or the response itself: prices are often filled in by page scripts
+        // from data in the response, such as "currentPrice":228
+        const body = (await store.response(ix)).body.toString('utf8');
+        const shown = /html/i.test(ix.entry.mime) ? cheerio.load(body)('body').text() : '';
+        const want = squash(r.source.text);
+        if (!squash(shown).includes(want) && !squash(body).includes(want))
+            errors.push(`require[${i}] (${r.field}): "${r.source.text}" does not appear on ${r.source.url}`);
+    }
+    return errors;
+}
+const squash = (s) => s.replace(/\s+/g, ' ').trim().toLowerCase();
+//# sourceMappingURL=grading.js.map

package/dist/src/grading.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grading.js","sourceRoot":"","sources":["../../src/grading.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,MAAY,OAAO,oCAAgB;AACnC,gDAAwB;AACxB,0CAAuC;AACvC,gDAAoD;AA0DpD,MAAM,KAAK,GAAgB,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;AAClE,MAAM,IAAI,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AACvD,MAAM,EAAE,GAAG,mBAAmB,CAAC;AAE/B,sBAA6B,IAAY;IACvC,MAAM,KAAK,GAAG,cAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IAC1C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACnH,OAAO,KAAgB,CAAC;AAC1B,CAAC;AAED,6EAA6E;AAC7E,yBAAgC,CAAU;IACxC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACvH,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;IAC/B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IAC9H,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC/C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,wBAAwB,IAAI,2CAA2C,CAAC,CAAC;QACzG,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,WAAW,IAAI,yBAAyB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3G,IAAI,CAAC,EAAE,WAAW,KAAK,SAAS,IAAI,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ;YAAE,MAAM,CAAC,IAAI,CAAC,WAAW,IAAI,yDAAyD,CAAC,CAAC;IAC/J,CAAC;IACD,MAAM,IAAI,GAAG,CAAI,GAAW,EAAE,CAAkB,EAAO,EAAE;QACvD,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,iBAAiB,CAAC,CAAC;QAC/E,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnC,CAAC,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAEtH,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACvB,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC;QAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAC3B,IAAI,CAAC,CAAC;YAAE,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,EAAE,KAAK,qBAAqB,CAAC,CAAC;QAC3E,MAAM,KAAK,GAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,CAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;QAC1G,IAAI,CAAC,KAAK,CAAC,MAAM;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,4CAA4C,CAAC,CAAC;QAClF,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,SAAS,IAAI,CAAC,CAAC,GAAG,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,8CAA8C,CAAC,CAAC;QACnM,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC;QAC1D,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAU;gBAAE,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ;oBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,2BAA2B,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC;YACzK,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,+BAA+B,CAAC,CAAC;YAChF,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,2CAA2C,CAAC,CAAC;QACzJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,CAAC,GAAG,KAAK,SAAS,IAAI,CAAC,CAAC,GAAG,KAAK,SAAS,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,0DAA0D,CAAC,CAAC;YAC1J,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,+CAA+C,CAAC,CAAC;YACnJ,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,wCAAwC,CAAC,CAAC;QAC9I,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,mCAAmC,CAAC,CAAC;QACpI,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,GAAG,KAAK,QAAQ,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,wDAAwD,CAAC,CAAC;IACnP,CAAC,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,MAAM,EAAE,GAAG,CAAC,EAAU,EAAE,CAAU,EAAE,EAAE;QACpC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,8CAA8C,CAAC,CAAC;aACtG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,iBAAiB,CAAC,CAAC;;YAC9D,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;IACF,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACtB,MAAM,EAAE,GAAG,UAAU,CAAC,GAAG,CAAC;QAC1B,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACd,IAAI,OAAO,CAAC,EAAE,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAC;QACrG,IAAI,CAAC,EAAE,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,oCAAoC,CAAC,CAAC;QACvI,IAAI,CAAC,EAAE,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,QAAQ;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,0CAA0C,CAAC,CAAC;IAC1G,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrB,MAAM,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC;QACzB,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,wCAAwC,CAAC,CAAC;QACnF,KAAK,MAAM,CAAC,IAAI,KAAK;YAAE,IAAI,OAAO,CAAC,EAAE,GAAG,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,GAAG;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC;IACxG,CAAC,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACI,KAAK,uBAAuB,CAAU,EAAE,KAAkB;IAC/D,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACjD,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,SAAS;QACxB,MAAM,IAAI,GAAG,IAAA,4BAAgB,EAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACjF,2EAA2E;QAC3E,MAAM,EAAE,GAAG,IAAA,eAAM,EAAC,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACpE,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,0BAA0B,CAAC,CAAC;YACnF,SAAS;QACX,CAAC;QACD,4FAA4F;QAC5F,wDAAwD;QACxD,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC9D,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,wBAAwB,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IACtK,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC"}

package/dist/src/judge.d.ts

@@ -0,0 +1,25 @@
+/** The LLM that reads an agent's answer for grading (docs/grading.md). */
+export interface Judge {
+    /** How the model is reached ("api" or "claude-cli"), recorded with each result. */
+    readonly via: string;
+    readonly model: string;
+    /** Asks once; the reply follows `schema`. Returns it and the model that answered. */
+    ask(req: JudgeRequest): Promise<{
+        output: unknown;
+        model: string;
+    }>;
+}
+export interface JudgeRequest {
+    system: string;
+    prompt: string;
+    schema: Record<string, unknown>;
+}
+export declare const DEFAULT_MODEL = "claude-opus-5-5";
+/**
+ * "api" (the Claude API, with ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN) or "cli" (a local
+ * `claude` that is signed in). Without a choice: the API when a key is set, otherwise the CLI.
+ */
+export declare function pickJudge(kind?: string, model?: string): Judge;
+export declare function apiJudge(model?: string): Judge;
+/** `claude -p` with no tools, run outside any project so no project instructions are read. */
+export declare function cliJudge(model?: string, bin?: string): Judge;

package/dist/src/judge.js

@@ -0,0 +1,119 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_MODEL = void 0;
+exports.pickJudge = pickJudge;
+exports.apiJudge = apiJudge;
+exports.cliJudge = cliJudge;
+const node_child_process_1 = require("node:child_process");
+const os = __importStar(require("node:os"));
+const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
+exports.DEFAULT_MODEL = 'claude-opus-5-5';
+/**
+ * "api" (the Claude API, with ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN) or "cli" (a local
+ * `claude` that is signed in). Without a choice: the API when a key is set, otherwise the CLI.
+ */
+function pickJudge(kind, model = exports.DEFAULT_MODEL) {
+    const k = kind ?? (process.env.ANTHROPIC_API_KEY || process.env.ANTHROPIC_AUTH_TOKEN ? 'api' : 'cli');
+    if (k === 'api')
+        return apiJudge(model);
+    if (k === 'cli')
+        return cliJudge(model);
+    throw new Error(`unknown judge "${kind}" (api or cli)`);
+}
+function apiJudge(model = exports.DEFAULT_MODEL) {
+    const client = new sdk_1.default();
+    return {
+        via: 'api',
+        model,
+        async ask({ system, prompt, schema }) {
+            const msg = await client.beta.messages
+                .stream({
+                model,
+                max_tokens: 16000,
+                system,
+                messages: [{ role: 'user', content: prompt }],
+                output_config: { effort: 'medium', format: { type: 'json_schema', schema } },
+                // a declined request is re-run on Anthropic's recommended fallback model
+                betas: ['server-side-fallback-2026-07-01'],
+                fallbacks: 'default',
+            })
+                .finalMessage();
+            if (msg.stop_reason === 'refusal')
+                throw new Error(`the grader declined: ${msg.stop_details?.explanation ?? 'no reason given'}`);
+            if (msg.stop_reason === 'max_tokens')
+                throw new Error('the grader ran out of output tokens');
+            const text = msg.content.flatMap((b) => (b.type === 'text' ? [b.text] : [])).join('');
+            return { output: JSON.parse(text), model: msg.model };
+        },
+    };
+}
+/** `claude -p` with no tools, run outside any project so no project instructions are read. */
+function cliJudge(model = exports.DEFAULT_MODEL, bin = process.env.PAB_CLAUDE_BIN ?? 'claude') {
+    return {
+        via: 'claude-cli',
+        model,
+        async ask({ system, prompt, schema }) {
+            const args = ['-p', '--output-format', 'json', '--model', model, '--tools', '', '--strict-mcp-config', '--no-session-persistence', '--system-prompt', system, '--json-schema', JSON.stringify(schema)];
+            const out = await run(bin, args, prompt);
+            let reply;
+            try {
+                reply = JSON.parse(out);
+            }
+            catch {
+                throw new Error(`${bin} -p did not return JSON: ${out.slice(0, 300)}`);
+            }
+            if (reply.is_error || reply.structured_output === undefined)
+                throw new Error(`${bin} -p failed: ${(reply.result ?? out).slice(0, 300)}`);
+            return { output: reply.structured_output, model: Object.keys(reply.modelUsage ?? {})[0] ?? model };
+        },
+    };
+}
+function run(bin, args, input) {
+    return new Promise((resolve, reject) => {
+        const child = (0, node_child_process_1.spawn)(bin, args, { cwd: os.tmpdir(), stdio: ['pipe', 'pipe', 'pipe'], timeout: 10 * 60_000 });
+        let out = '';
+        let err = '';
+        child.stdout.on('data', (d) => (out += d));
+        child.stderr.on('data', (d) => (err += d));
+        child.on('error', (e) => reject(new Error(`cannot run ${bin}: ${e.message}`)));
+        child.on('close', (code) => (code === 0 ? resolve(out) : reject(new Error(`${bin} exited with ${code}: ${(err || out).slice(0, 300)}`))));
+        child.stdin.end(input);
+    });
+}
+//# sourceMappingURL=judge.js.map

package/dist/src/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/judge.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2DAA2C;AAC3C,MAAY,EAAE,oCAAgB;AAC9B,4DAA0C;AAiB7B,QAAA,aAAa,GAAG,iBAAiB,CAAC;AAE/C;;;GAGG;AACH,mBAA0B,IAAa,EAAE,KAAK,GAAG,QAAA,aAAa;IAC5D,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACtG,IAAI,CAAC,KAAK,KAAK;QAAE,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,KAAK;QAAE,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,IAAI,KAAK,CAAC,kBAAkB,IAAI,gBAAgB,CAAC,CAAC;AAC1D,CAAC;AAED,kBAAyB,KAAK,GAAG,QAAA,aAAa;IAC5C,MAAM,MAAM,GAAG,IAAI,aAAS,EAAE,CAAC;IAC/B,OAAO;QACL,GAAG,EAAE,KAAK;QACV,KAAK;QACL,KAAK,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE;YAClC,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ;iBACnC,MAAM,CAAC;gBACN,KAAK;gBACL,UAAU,EAAE,KAAK;gBACjB,MAAM;gBACN,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;gBAC7C,aAAa,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE;gBAC5E,yEAAyE;gBACzE,KAAK,EAAE,CAAC,iCAAiC,CAAC;gBAC1C,SAAS,EAAE,SAAS;aACrB,CAAC;iBACD,YAAY,EAAE,CAAC;YAClB,IAAI,GAAG,CAAC,WAAW,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,YAAY,EAAE,WAAW,IAAI,iBAAiB,EAAE,CAAC,CAAC;YACjI,IAAI,GAAG,CAAC,WAAW,KAAK,YAAY;gBAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;YAC7F,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACtF,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC;QACxD,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8FAA8F;AAC9F,kBAAyB,KAAK,GAAG,QAAA,aAAa,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,QAAQ;IAC1F,OAAO;QACL,GAAG,EAAE,YAAY;QACjB,KAAK;QACL,KAAK,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE;YAClC,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,iBAAiB,EAAE,MAAM,EAAE,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YACvM,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;YACzC,IAAI,KAAiH,CAAC;YACtH,IAAI,CAAC;gBACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,4BAA4B,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACzE,CAAC;YACD,IAAI,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,iBAAiB,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACzI,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,iBAAiB,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;QACrG,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,GAAG,CAAC,GAAW,EAAE,IAAc,EAAE,KAAa;IACrD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,IAAA,0BAAK,EAAC,GAAG,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;QAC5G,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3C,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3C,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/E,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,GAAG,gBAAgB,IAAI,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1I,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/src/paths.d.ts

@@ -0,0 +1,2 @@
+/** Root of this package: the repository checkout, or node_modules/personal-agent-benchmark. */
+export declare const PACKAGE_ROOT: string;

package/dist/src/paths.js

@@ -0,0 +1,40 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PACKAGE_ROOT = void 0;
+const path = __importStar(require("node:path"));
+/** Root of this package: the repository checkout, or node_modules/personal-agent-benchmark. */
+exports.PACKAGE_ROOT = path.resolve(__dirname, '..', '..');
+//# sourceMappingURL=paths.js.map

package/dist/src/paths.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"paths.js","sourceRoot":"","sources":["../../src/paths.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,MAAY,IAAI,sCAAkB;AAElC,+FAA+F;AAClF,QAAA,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC"}

package/dist/src/replay/clock.d.ts

@@ -0,0 +1,19 @@
+import type { HeaderList } from '../core/types';
+/**
+ * Page clock shim: shifts `Date` in replayed HTML documents to the recording time, so countdowns
+ * and "deal ends in" logic behave as they did during capture. Time keeps flowing from that point.
+ * Done in the proxy rather than with libfaketime, which breaks Chromium's CDP in practice and
+ * would not reach agents that bring their own browser.
+ */
+export declare function clockScript(recordedMs: number): string;
+/** Inserts the shim at the top of <head> (or the document) and lets it through any CSP. */
+export declare function injectClock(html: string, headers: HeaderList, recordedMs: number): {
+    html: string;
+    headers: HeaderList;
+};
+/**
+ * Lets the shim through script-src / script-src-elem. When a directive allows 'unsafe-inline' and
+ * has no hash or nonce, it is left alone: adding a hash would switch 'unsafe-inline' off for the
+ * page's own inline scripts.
+ */
+export declare function allowScriptHash(csp: string, hash: string): string;

package/dist/src/replay/clock.js

@@ -0,0 +1,60 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.clockScript = clockScript;
+exports.injectClock = injectClock;
+exports.allowScriptHash = allowScriptHash;
+const node_crypto_1 = require("node:crypto");
+/**
+ * Page clock shim: shifts `Date` in replayed HTML documents to the recording time, so countdowns
+ * and "deal ends in" logic behave as they did during capture. Time keeps flowing from that point.
+ * Done in the proxy rather than with libfaketime, which breaks Chromium's CDP in practice and
+ * would not reach agents that bring their own browser.
+ */
+function clockScript(recordedMs) {
+    return ('(function(){if(window.__rrClock)return;window.__rrClock=1;' +
+        `var D=Date,o=${recordedMs}-D.now();` +
+        'function F(){if(!(this instanceof F))return new D(D.now()+o).toString();' +
+        'if(arguments.length===0)return new D(D.now()+o);' +
+        'return new (Function.prototype.bind.apply(D,[null].concat([].slice.call(arguments))))();}' +
+        'F.prototype=D.prototype;F.now=function(){return D.now()+o};F.parse=D.parse;F.UTC=D.UTC;' +
+        "try{Object.defineProperty(F,'name',{value:'Date'})}catch(e){}window.Date=F;})();");
+}
+/** Inserts the shim at the top of <head> (or the document) and lets it through any CSP. */
+function injectClock(html, headers, recordedMs) {
+    if (html.includes('__rrClock'))
+        return { html, headers };
+    const script = clockScript(recordedMs);
+    const tag = `<script>${script}</script>`;
+    const head = /<head[^>]*>/i.exec(html);
+    const out = head ? html.slice(0, head.index + head[0].length) + tag + html.slice(head.index + head[0].length) : tag + html;
+    const hash = `'sha256-${(0, node_crypto_1.createHash)('sha256').update(script).digest('base64')}'`;
+    const patched = headers.map(([k, v]) => (k.toLowerCase() === 'content-security-policy' ? [k, allowScriptHash(v, hash)] : [k, v]));
+    return { html: out, headers: patched };
+}
+/**
+ * Lets the shim through script-src / script-src-elem. When a directive allows 'unsafe-inline' and
+ * has no hash or nonce, it is left alone: adding a hash would switch 'unsafe-inline' off for the
+ * page's own inline scripts.
+ */
+function allowScriptHash(csp, hash) {
+    const directives = csp.split(';').map((d) => d.trim()).filter(Boolean);
+    if (!directives.some((d) => /^script-src(-elem)?(\s|$)/i.test(d))) {
+        const fallback = directives.find((d) => /^default-src(\s|$)/i.test(d));
+        // only default-src governs scripts: derive a script-src from it that also allows the shim
+        if (fallback && !/'unsafe-inline'/i.test(fallback))
+            directives.push(fallback.replace(/^default-src/i, 'script-src') + ' ' + hash);
+        return directives.join('; ');
+    }
+    return directives
+        .map((d) => {
+        const directive = d.trim();
+        if (!/^script-src(-elem)?(\s|$)/i.test(directive))
+            return directive;
+        const hasHashOrNonce = /'(nonce|sha256|sha384|sha512)-/i.test(directive);
+        if (/'unsafe-inline'/i.test(directive) && !hasHashOrNonce)
+            return directive;
+        return `${directive} ${hash}`;
+    })
+        .join('; ');
+}
+//# sourceMappingURL=clock.js.map

package/dist/src/replay/clock.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"clock.js","sourceRoot":"","sources":["../../../src/replay/clock.ts"],"names":[],"mappings":";;;;;AAAA,6CAAyC;AAGzC;;;;;GAKG;AACH,qBAA4B,UAAkB;IAC5C,OAAO,CACL,4DAA4D;QAC5D,gBAAgB,UAAU,WAAW;QACrC,0EAA0E;QAC1E,kDAAkD;QAClD,2FAA2F;QAC3F,yFAAyF;QACzF,kFAAkF,CACnF,CAAC;AACJ,CAAC;AAED,2FAA2F;AAC3F,qBAA4B,IAAY,EAAE,OAAmB,EAAE,UAAkB;IAC/E,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IACzD,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;IACvC,MAAM,GAAG,GAAG,WAAW,MAAM,WAAW,CAAC;IACzC,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC;IAC3H,MAAM,IAAI,GAAG,WAAW,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;IAChF,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAoB,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACpJ,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AACzC,CAAC;AAED;;;;GAIG;AACH,yBAAgC,GAAW,EAAE,IAAY;IACvD,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACvE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,4BAA4B,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,0FAA0F;QAC1F,IAAI,QAAQ,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC;YAAE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,EAAE,YAAY,CAAC,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC;QAClI,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,UAAU;SACd,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,4BAA4B,CAAC,IAAI,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;QACpE,MAAM,cAAc,GAAG,iCAAiC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzE,IAAI,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,SAAS,CAAC;QAC5E,OAAO,GAAG,SAAS,IAAI,IAAI,EAAE,CAAC;IAChC,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC"}

package/dist/src/replay/engine.d.ts

@@ -0,0 +1,53 @@
+import { Bundle } from '../core/bundle';
+import { type NormalizedRequest } from '../core/normalize';
+import { RuleSet } from '../core/rules';
+import { RecordStore } from '../core/store';
+import type { Manifest, ReplayRequest, ReplayResponse, ResourceClass } from '../core/types';
+export declare const CONTROL_HOST = "rr.control";
+export interface EngineOptions {
+    logFile?: string;
+    /** Extra rule directories layered over the bundle's own rules (later wins). */
+    rulesDirs?: string[];
+    /** "recorded" (default) shifts page clocks to the recording time; "off" leaves them alone. */
+    clock?: 'recorded' | 'off';
+}
+export interface ReplayStats {
+    requests: number;
+    bySource: Record<string, number>;
+    byClass: Record<ResourceClass, {
+        total: number;
+        hit: number;
+        miss: number;
+    }>;
+    misses: Record<string, {
+        count: number;
+        resourceClass: ResourceClass;
+        url: string;
+    }>;
+}
+/**
+ * Answers every request from a recording bundle. It never opens a network connection:
+ * misses are answered locally and logged.
+ */
+export declare class ReplayEngine {
+    readonly bundle: Bundle;
+    readonly rules: RuleSet;
+    readonly store: RecordStore;
+    readonly manifest: Manifest;
+    private stats;
+    private readonly logFile?;
+    private readonly clockMs?;
+    constructor(bundleDir: string, opts?: EngineOptions);
+    getStats(): ReplayStats;
+    reset(): void;
+    handle(req: ReplayRequest): Promise<ReplayResponse>;
+    private resolve;
+    private missResponse;
+    private cachedNotFound;
+    private notFoundPage;
+    /** Loads each site's recorded 404 page once; call before serving. */
+    warmUp(): Promise<void>;
+    private control;
+    private record;
+}
+export declare function classify(req: ReplayRequest, norm: NormalizedRequest): ResourceClass;