personal-agent-benchmark 0.0.0-stage → 0.2.0

package/dist/src/tasks.js

@@ -0,0 +1,152 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Task = exports.Repo = void 0;
+exports.resolveTarget = resolveTarget;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const yaml_1 = __importDefault(require("yaml"));
+const archive_1 = require("./core/archive");
+const paths_1 = require("./paths");
+class Repo {
+    root;
+    constructor(root) {
+        this.root = root;
+    }
+    /** $PAB_REPO, or the checkout this code runs from. */
+    static default() {
+        return new Repo(path.resolve(process.env.PAB_REPO ?? paths_1.PACKAGE_ROOT));
+    }
+    get sitesDir() { return path.join(this.root, 'sites'); }
+    get tasksDir() { return path.join(this.root, 'tasks'); }
+    /** Unpacked bundles and replay logs ($PAB_CACHE, default .cache/ in the repository). */
+    get cacheDir() { return path.resolve(process.env.PAB_CACHE ?? path.join(this.root, '.cache')); }
+    taskIds() {
+        if (!fs.existsSync(this.tasksDir))
+            return [];
+        return fs
+            .readdirSync(this.tasksDir)
+            .filter((name) => fs.existsSync(path.join(this.tasksDir, name, 'task.yaml')))
+            .sort();
+    }
+    task(id) {
+        const dir = path.join(this.tasksDir, id);
+        if (!fs.existsSync(path.join(dir, 'task.yaml')))
+            throw new Error(`no task "${id}" in ${this.tasksDir}`);
+        return new Task(this, dir);
+    }
+}
+exports.Repo = Repo;
+class Task {
+    repo;
+    dir;
+    constructor(repo, dir) {
+        this.repo = repo;
+        this.dir = dir;
+    }
+    get id() { return path.basename(this.dir); }
+    get cardPath() { return path.join(this.dir, 'task.yaml'); }
+    get recordingDir() { return path.join(this.dir, 'recording'); }
+    get rulesDir() { return path.join(this.dir, 'rules'); }
+    /** grading.yaml: how answers are scored (see grading.ts). */
+    get gradingPath() { return path.join(this.dir, 'grading.yaml'); }
+    card() {
+        const card = yaml_1.default.parse(fs.readFileSync(this.cardPath, 'utf8'));
+        return { ...card, id: card?.id ?? this.id };
+    }
+    /** Files of the recording; empty when the task has none yet. */
+    recordingFiles() {
+        return (0, archive_1.recordingFiles)(this.recordingDir);
+    }
+    /** Unpacks the recording into the cache (once per recording content) and returns its directory. */
+    async bundleDir() {
+        const files = this.recordingFiles();
+        if (!files.length)
+            throw new Error(`task ${this.id} has no recording (${this.recordingDir})`);
+        const digest = (0, archive_1.hashParts)(files).slice(0, 16);
+        const bundlesDir = path.join(this.repo.cacheDir, 'bundles');
+        const dest = path.join(bundlesDir, `${this.id}-${digest}`);
+        if (fs.existsSync(path.join(dest, 'manifest.json')))
+            return dest;
+        const tmp = `${dest}.tmp-${process.pid}`;
+        fs.rmSync(tmp, { recursive: true, force: true });
+        try {
+            await (0, archive_1.unpackRecording)(this.recordingDir, tmp);
+            fs.rmSync(dest, { recursive: true, force: true });
+            fs.renameSync(tmp, dest);
+        }
+        catch (err) {
+            fs.rmSync(tmp, { recursive: true, force: true });
+            throw err;
+        }
+        // older unpacked versions of this task
+        for (const name of fs.readdirSync(bundlesDir)) {
+            if (name.startsWith(`${this.id}-`) && path.join(bundlesDir, name) !== dest && !name.includes('.tmp-')) {
+                fs.rmSync(path.join(bundlesDir, name), { recursive: true, force: true });
+            }
+        }
+        return dest;
+    }
+    async target() {
+        return {
+            name: this.id,
+            bundleDir: await this.bundleDir(),
+            rulesDirs: [this.repo.sitesDir, this.rulesDir],
+            task: this,
+        };
+    }
+}
+exports.Task = Task;
+/**
+ * Resolves a command-line argument to a replay target: a task id, a task directory, or a bundle
+ * directory (one with manifest.json, replayed with its own rules/).
+ */
+async function resolveTarget(arg, repo = Repo.default()) {
+    const asPath = path.resolve(arg);
+    if (fs.existsSync(path.join(asPath, 'manifest.json'))) {
+        return { name: path.basename(asPath), bundleDir: asPath, rulesDirs: [] };
+    }
+    if (fs.existsSync(path.join(asPath, 'task.yaml'))) {
+        // a task inside a repository uses that repository's sites/; a lone task directory uses ours
+        const parent = path.dirname(asPath);
+        const owner = path.basename(parent) === 'tasks' && fs.existsSync(path.join(parent, '..', 'sites')) ? new Repo(path.dirname(parent)) : repo;
+        return new Task(owner, asPath).target();
+    }
+    return repo.task(arg).target();
+}
+//# sourceMappingURL=tasks.js.map

package/dist/src/tasks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tasks.js","sourceRoot":"","sources":["../../src/tasks.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,MAAY,EAAE,oCAAgB;AAC9B,MAAY,IAAI,sCAAkB;AAClC,gDAAwB;AACxB,4CAA4E;AAC5E,mCAAuC;AA4BvC;IACuB,IAAI;IAAzB,YAAqB,IAAY;oBAAZ,IAAI;IAAW,CAAC;IAErC,sDAAsD;IACtD,MAAM,CAAC,OAAO;QACZ,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,oBAAY,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,QAAQ,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAChE,IAAI,QAAQ,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAChE,wFAAwF;IACxF,IAAI,QAAQ,KAAa,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAExG,OAAO;QACL,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,CAAC;QAC7C,OAAO,EAAE;aACN,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC;aAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;aAC5E,IAAI,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,CAAC,EAAU;QACb,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,YAAY,EAAE,QAAQ,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxG,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC7B,CAAC;CACF;;AAED;IACuB,IAAI;IAAiB,GAAG;IAA7C,YAAqB,IAAU,EAAW,GAAW;oBAAhC,IAAI;mBAAiB,GAAG;IAAW,CAAC;IAEzD,IAAI,EAAE,KAAa,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACpD,IAAI,QAAQ,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;IACnE,IAAI,YAAY,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;IACvE,IAAI,QAAQ,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/D,6DAA6D;IAC7D,IAAI,WAAW,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC;IAEzE,IAAI;QACF,MAAM,IAAI,GAAG,cAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAa,CAAC;QAC5E,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;IAC9C,CAAC;IAED,gEAAgE;IAChE,cAAc;QACZ,OAAO,IAAA,wBAAc,EAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;IAED,mGAAmG;IACnG,KAAK,CAAC,SAAS;QACb,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,EAAE,sBAAsB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QAC9F,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC,CAAC;QAC3D,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QACjE,MAAM,GAAG,GAAG,GAAG,IAAI,QAAQ,OAAO,CAAC,GAAG,EAAE,CAAC;QACzC,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC;YACH,MAAM,IAAA,yBAAe,EAAC,IAAI,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;YAC9C,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAClD,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACjD,MAAM,GAAG,CAAC;QACZ,CAAC;QACD,uCAAuC;QACvC,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACtG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,MAAM;QACV,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,EAAE;YACb,SAAS,EAAE,MAAM,IAAI,CAAC,SAAS,EAAE;YACjC,SAAS,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC;YAC9C,IAAI,EAAE,IAAI;SACX,CAAC;IACJ,CAAC;CACF;;AAED;;;GAGG;AACI,KAAK,wBAAwB,GAAW,EAAE,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE;IACpE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,EAAE,CAAC;QACtD,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAC3E,CAAC;IACD,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC;QAClD,4FAA4F;QAC5F,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,OAAO,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3I,OAAO,IAAI,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;AACjC,CAAC"}

package/docs/bundle-format.md

@@ -0,0 +1,114 @@
+# Bundle format
+
+A bundle is one recording of a task (or several merged into one). It holds only what replay
+reads. In this repository it travels as `tasks/<id>/recording/` (the published layout, below),
+which `pab` unpacks into `.cache/bundles/<id>-<hash>/` on first use.
+
+```text
+manifest.json            the few facts replay needs (below)
+archive/data.warc        every HTTP exchange (WARC 1.1, uncompressed records)
+indexes/entries.jsonl    replay index: one line per exchange, with the request body
+pages/pages.jsonl        top-level pages visited while recording
+```
+
+Web-archive tools such as ReplayWeb.page read WACZ files, which also carry a CDXJ index and a
+`datapackage.json`. Replay does not use them, so bundles do not store them; the recording tool
+builds them when it exports a WACZ. Bundles written before this change still contain both files
+and unpack as before.
+
+## manifest.json
+
+```json
+{
+  "task_id": "amazon-airpods",
+  "captured_from": "2026-10-22T10:40:57.458Z",
+  "captured_to": "2026-10-22T12:50:41.949Z",
+  "not_found": { "amazon": "https://www.amazon.com/rr-not-found-mgxy1k2a" },
+  "locale": "en-US",
+  "timezone": "America/New_York"
+}
+```
+
+| Field | Used for |
+|---|---|
+| `task_id` | which `tasks/<id>/` the recording belongs to |
+| `captured_from` | page clocks start here during replay |
+| `captured_to` | with `captured_from`, tells merged recordings apart |
+| `not_found` | each site's own 404 page, answered for documents replay cannot find |
+| `locale`, `timezone` | the replay browser's language and time zone |
+| `format` | layout version of the published recording (below); only in `recording/manifest.json` |
+
+Nothing else is recorded: not who recorded it, where from, with which browser, or request
+counts. `pab audit` fails a manifest with any other field. Recordings made before 2026-10-26
+also have `start_url`; replay ignores it, the audit lets it pass, and the next import of the
+task drops it. Replay opens the first page opened while recording unless given `--url`.
+
+Rules are not part of the published bundle; they live next to it in the task
+directory, so they can be reviewed and changed without touching the recording.
+
+## Published layout (format 2)
+
+```text
+tasks/<id>/recording/
+  manifest.json        the manifest above, plus "format": 2
+  pages.jsonl          pages/pages.jsonl
+  entries.jsonl.gz     indexes/entries.jsonl, gzip-compressed
+  warc.tar.gz          archive/data.warc in a gzip-compressed tar
+                       (warc.tar.gz.000, .001, ... when larger than 45 MB)
+```
+
+The WARC is almost all of a recording (the Sony task: 74 MB of 78 MB) and follows the WARC
+standard, so it is stored apart from the small files whose layout is ours. The tar is written
+with fixed timestamps, so the same WARC always gives the same bytes. WARC records are stored
+uncompressed inside it: one gzip stream over the whole file compresses far better than
+per-record gzip (the Amazon task: 33 MB instead of 39 MB). Archives larger than 45 MB are split
+into numbered parts (GitHub warns above 50 MB and refuses 100 MB) and read back in order.
+Extraction accepts only `archive/data.warc`, and only as a regular file.
+
+### Changing the format
+
+`format` is the contract between this repository and the tool that writes recordings into it.
+Replay reads exactly one format, `RECORDING_FORMAT` in `src/core/archive.ts`, and refuses any
+other with an error naming both. A change to the layout or to the files in it bumps the number,
+and the same change rewrites every task to the new format with a script, so replay never keeps
+readers for old formats. When only the small files change, the rewrite leaves `warc.tar.gz` as
+it was and git stores nothing new for it.
+
+Format 1 was one `bundle.tar.gz` (split the same way) holding all four files; format 2 replaced
+it on 2026-10-26.
+
+## WARC records
+
+Each exchange is two records: `response` (or `revisit`) first, then `request`, whose
+`WARC-Concurrent-To` points at the response.
+
+- Response bodies are stored **decoded**. `Content-Encoding` and `Transfer-Encoding` are removed
+  and `Content-Length` matches the stored body; the original encoding is kept in the WARC header
+  `RR-Original-Content-Encoding`.
+- Identical payloads of 1 KB or more are stored once; later copies are `revisit` records
+  (`WARC-Profile: identical-payload-digest`) and `revisitOf` in `entries.jsonl` points at the
+  original.
+- Requests carry no `Cookie` or `Authorization` header; every `Set-Cookie` value is `REDACTED`.
+  `pab audit` verifies both.
+- Every header line is a header: a header sent several times is stored once per value. Recordings
+  made with record-replay before 0.5.5 stored such a header as one value joined by newlines, so
+  each further value is a line of its own; readers take those lines as values of the header on the
+  line before, and `pab audit` fails a bundle that has any.
+- Custom WARC headers: `RR-Exchange-Id` (the id in `entries.jsonl`), `RR-Resource-Type`.
+
+## entries.jsonl
+
+```json
+{"id":"e000123","ts":"2026-10-22T10:41:05.120Z","method":"GET","url":"https://www.amazon.com/Apple-.../dp/B0FQFB8FMG/ref=sr_1_1?...","status":200,"mime":"text/html","digest":"<sha256>","size":812345,"reqOffset":1234,"reqLength":456,"respOffset":789,"respLength":101112}
+```
+
+POST bodies are kept base64-encoded in `reqBody`. Matching keys are not stored: the replay engine
+computes them from the current rules when it starts, so changing a rule never requires a new
+recording.
+
+## pages.jsonl
+
+Pages visited while recording, in order. `fetched` says how the address was reached: `GET` (a
+document load), `POST` (a form result) or `none` (rewritten by page script). `note:
+"not-found-probe"` marks a visit to a path that cannot exist, made to record the site's own 404
+page; replay serves that page for documents it cannot find.

package/docs/deploy.md

@@ -0,0 +1,78 @@
+# Running replays in containers
+
+This repository ships the replay as images and says what a platform running agents against it
+must ensure. How the platform schedules a replay (Pods, Services, network policies) belongs to
+that platform's integration, not here.
+
+## Images
+
+Two images per task, built on the machine that runs the replay; nothing is pushed to a registry.
+
+| Image | What is in it | Built again when |
+|---|---|---|
+| `pab-replay:<digest>` | the replay service (`deploy/Dockerfile`): whistle, the replay plugin, `sites/` | the code or shared site rules change |
+| `pab-task:<id>-<digest>` | one task directory (`deploy/task/Dockerfile`): recording, `rules/`, `task.yaml`, `grading.yaml` | that task changes |
+
+```bash
+sh deploy/build-images.sh tasks/<id>    # builds what is missing, prints both image names
+```
+
+Tags are digests of what goes into each image, so a run can record exactly which code and which
+recording it used. Old tags are not removed. The task image keeps the recording packed; the
+replay service unpacks it when it starts (about a second for a 75 MB recording, 214 MB
+unpacked).
+
+## Running one replay
+
+- Run the task image once with a volume at `/out`: it copies the task there and exits.
+- Run the replay image with that volume at `/task`. It serves the proxy on port 8899 and writes
+  every request, and how it was answered, to `/logs/replay.jsonl`. That log is what `pab grade`
+  scores the run from (docs/grading.md), so keep it after the run.
+- One replay serves one task for one run: start a fresh one per run.
+
+## What the platform must ensure
+
+The replay itself:
+
+- has no network access beyond answering the agent's browser (and whatever reads its log and
+  `http://rr.control/stats`). The proxy never opens a connection, but nothing should let it.
+
+The agent's browser, so that the recording rather than the internet answers it:
+
+1. sends everything through the proxy: `--proxy-server=http://<replay host>:8899`, with no
+   bypass list beyond loopback (some browser images add private ranges to `--proxy-bypass-list`
+   unless it is given);
+2. turns QUIC off (`--disable-quic`): QUIC runs over UDP and does not go through an HTTP proxy;
+3. accepts the proxy's certificates (`--ignore-certificate-errors`, or trust whistle's root CA):
+   the proxy answers HTTPS itself;
+4. can reach nothing but the proxy (and DNS to find it);
+5. starts from a fresh profile each run (no cache or service workers from an earlier run);
+6. preferably uses the recording's time zone and language (`timezone` and `locale` in the
+   bundle's `manifest.json`).
+
+Page clocks need nothing from the browser (see below). Agent tools that fetch pages without the
+browser (HTTP fetch, search APIs) bypass the proxy: route them through it (`HTTPS_PROXY`) or turn
+them off for replay runs.
+
+`deploy/browser/` (`pab-browser`) is Chromium set up this way. It stands in for the agent's
+browser in the local check.
+
+## Page clocks
+
+The replay service injects a small script at the top of every HTML page that shifts `Date` to the
+recording time (`captured_from` in the manifest) and lets it run on from there; pages with a CSP
+get the script's hash added to `script-src`. `RR_CLOCK=off` turns this off. libfaketime in the
+browser container was tried and makes Chromium's CDP handshake hang; it would also miss agents
+that bring their own browser.
+
+## Local check
+
+```bash
+npm run build
+sh deploy/local-pod-test.sh tasks/<id> <URL>    # Docker internal network, no internet
+```
+
+It builds the images it lacks, runs the replay and the stand-in browser on a Docker network
+without internet access, drives the browser over CDP from a third container
+(`scripts/pod-smoke.ts`) to open `<URL>` (a page in the recording), and checks that the network
+has no way out.

package/docs/grading.md

@@ -0,0 +1,81 @@
+# Grading
+
+A personal agent answers in text, so a task is scored from its answer with an LLM and from what
+it did in the replay. Each task says how in `tasks/<id>/grading.yaml`, next to `task.yaml`. The
+grading requirements are written by the person who set the task; the recording tool turns them
+into this format when the recording is imported.
+
+## Format
+
+```yaml
+extract:                        # fields an LLM extracts from the agent's answer
+  price: { type: money, description: lowest price found, without shipping }
+  store: { type: text, description: the website selling at that price }
+
+require:                        # compared by the program; any failure scores 0
+  - field: price
+    equals: 278.00              # numbers: equals, or min/max; tolerance defaults to 0.01
+    source: { url: "https://www.bestbuy.com/site/sony-wh1000xm5/6505727.p", text: "$278.00" }
+  - field: store
+    one_of: ["Best Buy", "bestbuy.com"]   # text: equals, one_of or contains; case and spacing ignored
+
+checks:                         # yes/no questions an LLM answers about the answer
+  - id: right_product
+    question: Is the product in the answer a new, black Sony WH-1000XM5 (not renewed, used or another model)?
+    required: true              # "no" scores 0; the default when no weight is given
+  - id: says_shipping
+    question: Does the answer say whether shipping is free?
+    weight: 1                   # partial credit
+
+trace:                          # requests the agent made in the replay
+  - id: no_order
+    must_not: { method: POST, url: "*/checkout/place*" }
+  - id: added_to_cart
+    must: { method: POST, url: "*/cart/add*", hit: true }   # hit: answered from the recording
+```
+
+| Field | Types | Tests |
+|---|---|---|
+| `money`, `number` | numbers | `equals` (with `tolerance`), `min`, `max`, `one_of` |
+| `text` | text | `equals`, `one_of`, `contains` |
+| `boolean` | true / false | `equals` |
+
+`source` says where the expected value appears in the recording: a recorded page (or API
+response) and the text shown on it, or the text as it is in the response when page scripts fill
+the value in (for example `"currentPrice":228`). Import checks that the page is in the recording
+and holds that text, so the answer key is something the agent could have read during replay.
+
+## Score
+
+- Required items are every `require` entry, every check without `weight`, and every `trace`
+  entry. If any of them fails, the score is 0.
+- Otherwise the score is the weight of the weighted checks that passed divided by their total
+  weight, or 1 when there are none.
+
+A grading needs at least one of `require`, `checks` or `trace`.
+
+## Run report
+
+`pab grade <task> --log <replay-log.jsonl> --answer <file>` scores one run and writes two files
+side by side, every time:
+
+- `result.json`: the score, the extracted fields, each item with its result and reason (expected
+  and actual values, the LLM's reason, the matching request), the model that graded, the answer,
+  and the pages the agent opened.
+- `report.html`: the same, as a page to read, rendered from `result.json`.
+
+Pages are what the agent opened in a browser tab (`sec-fetch-dest: document` in the replay log),
+not frames, scripts, styles or API calls. `pages.not_found` lists those that came back 404, one row
+per normalized key with the site, the route, the first URL and how many times it was opened:
+
+- `source: miss`: the recording has no copy. Recording the page lets a later run see it.
+- any other source: the site itself answered 404 while recording.
+
+Without `--answer`, or for a task without `grading.yaml`, nothing is scored but both files are
+still written with the page list. When the grader fails, the files are written with a note and the
+command exits with 1.
+
+The grader is Claude (`claude-opus-5-5`, `--model` to change). With `ANTHROPIC_API_KEY` or
+`ANTHROPIC_AUTH_TOKEN` set it calls the Claude API, where a declined request is re-run on
+Anthropic's recommended fallback model; otherwise it runs a signed-in local `claude` CLI.
+`--judge api|cli` picks one.

package/docs/site-rules.md

@@ -0,0 +1,114 @@
+# Site rules and request matching
+
+## The matching key
+
+```text
+key = method + host and normalized path + kept query parameters (sorted) + normalized body
+e.g. GET www.amazon.com/dp/B0FQFB8FMG
+     GET www.amazon.com/s?k=airpods pro
+     POST shop.test/cart/add #body:{"ASIN":"B0TEST","quantity":"1"}
+```
+
+The key ignores the scheme (http and https are the same request). Keys are computed when replay
+starts, from the rules in effect, so a rule change takes effect on the next replay without a new
+recording.
+
+## Where rules come from
+
+| File | Written by | Applies to |
+|---|---|---|
+| `sites/_default.yaml` | people | every site: generic tracking parameters, ad and analytics hosts |
+| `sites/<site>.yaml` | people | every task on that site |
+| `tasks/<id>/rules/<site>.auto.yaml` | the recording tool | this task; regenerated with each recording |
+| `tasks/<id>/rules/<site>.yaml` | people | this task |
+
+Hand-written rules win over inferred ones; the task's hand-written rules win over `sites/`
+(details in [tasks.md](tasks.md)).
+
+### What is inferred
+
+| Inferred | From |
+|---|---|
+| Excluded bot-check pages (`exclude`) | visible text or path of a captcha page, e.g. "I am human", `/checkpoint/` |
+| Volatile parameters (`drop_params`, `body_drop`) | values that differ between loads of the same path and look random or leave the response unchanged; timestamps near the recording time |
+| Page identity (`key` + `keep_params: []`) | an ID in the page's `<link rel="canonical">` (Amazon ASIN, numeric IDs, Target `A-…`): title slugs and tracking segments map to `/…/{id}` |
+| Searches (`kind: search`) | common query parameter names (`q`, `k`, `query`, `field-keywords`, ...); suggestions by `prefix` and similar |
+| Trackers (`block`) | near-empty responses on beacon-like paths (`events`, `batch`, `/px/`, `uedata`, ...); hosts that serve nothing else are blocked whole |
+
+What people usually write: the occasional fix.
+
+Effect on the Amazon task (same recording; self-check plus an agent run):
+
+| | generic rules only | + inferred | + hand-written |
+|---|---|---|---|
+| self-check: API requests matched | 52 / 159 | 42 / 42 | 42 / 42 |
+| agent searches with unrecorded wording | 404, no product | finds it, reads $199.00 | same |
+
+## Rule file
+
+```yaml
+site: amazon                      # rule name
+hosts: ["amazon.com", "*.amazon.com"]
+drop_params: [ref, qid, sr, "pd_rd_*"]   # query parameters ignored when matching (* wildcards)
+body_drop: ["*token*"]            # request body fields ignored when matching (recursive)
+block: ["fls-na.amazon.com", "www.amazon.com/1/batch/**"]   # answered with 204 (host, or host/path)
+exclude: ["www.amazon.com/checkpoint/**"]   # never stored, never replayed
+latest: ["*/gp/cart/*"]           # answered with the last recording (carts); _default.yaml covers */cart*, */checkout*
+xhr_not_found: { content_type: application/json, body: "{}" } # optional body for missed API calls
+endpoints:
+  - name: product
+    match: "GET /*/dp/{asin}/**"  # method optional; * one segment, ** any segments, {x} captures one
+    key: "/dp/{asin}"             # normalized path
+  - name: search
+    match: "GET /s"
+    host: "www.amazon.com"        # optional: only this host
+    keep_params: [k, page]        # only these parameters count (allow-list, beats drop_params)
+    kind: search                  # free text: matched by word-set similarity
+    text_param: k
+    similarity: 0.6
+  - name: add_to_cart
+    match: "POST /cart/add-to-cart/**"
+    kind: stateful                # changes state; a request with a body only matches exactly
+    body_keep: [ASIN, quantity]   # only these body fields count: another item or quantity misses
+  - name: promo
+    match: "POST /cart/promo"
+    kind: promo                   # known codes get their recorded answer; unknown codes get the
+    text_param: code              # answer recorded for invalid_code
+    invalid_code: BADCODE
+```
+
+Other endpoint fields: `fuzzy` (allow same-path matching with different parameters; on by default
+for GET, off for requests with a body) and `fuzzy_min` (share of parameters that must agree,
+default 0.5; 0 for static files).
+
+## Lookup order
+
+1. `block` rules: 204.
+2. Exact key. When a key was recorded more than once, the earliest copy wins ("browse first,
+   act later"), except for pages matching `latest` (carts, checkouts), where the last copy wins:
+   that is the state the task ended in.
+3. Free text: `search` finds the most similar recorded search; `promo` as above.
+4. Fuzzy: same method and path, the most query parameters in common (for `latest` pages, the
+   last of equally good matches).
+5. Miss:
+   - trackers: 204;
+   - images, fonts, CSS, JS: 404 with an empty body;
+   - documents: the site's own 404 page (recorded during capture), else a plain 404 page;
+   - API requests: 404 with `xhr_not_found`, or `{}` when JSON is accepted.
+6. CORS preflights (OPTIONS) that were not recorded are allowed.
+
+The request type comes from `Sec-Fetch-Dest`, falling back to `Accept` and the file extension.
+With `--log`, every request is logged as one JSON line: time, method, URL, key, type, source
+(exact/fuzzy/search/promo/blocked/preflight/miss), status, matched record, duration.
+
+## Fixing misses
+
+```bash
+node bin/pab.js inspect <task> "<url>"     # normalized key, and which record it hits
+node bin/pab.js replay <task> --log replay.jsonl && node bin/pab.js stats replay.jsonl
+```
+
+- The same path keeps missing with different parameters: add the parameter to `drop_params`.
+- Product URLs carry changing slugs and `ref` segments: map them with `match` + a `key` template.
+- A POST carries a fresh token each time: `body_drop` it, or list the fields that matter in
+  `body_keep`.

package/docs/tasks.md

@@ -0,0 +1,63 @@
+# Tasks
+
+One directory per task under `tasks/`:
+
+```text
+tasks/<id>/
+  task.yaml          the task card
+  recording/         the recording (docs/bundle-format.md)
+  grading.yaml       how answers are scored (docs/grading.md)
+  rules/             site rules for this task
+    <site>.auto.yaml   inferred from the recording; regenerated on every import, do not edit
+    <site>.yaml        written by people; corrections and additions
+```
+
+## task.yaml
+
+```yaml
+id: amazon-airpods
+title: Buy AirPods Pro 3 on Amazon
+prompt: >-                       # the instruction given to the agent
+  I want Apple AirPods Pro 3 from Amazon, shipped to New York (ZIP 10001). ...
+capture:                         # the browser's language and time zone while recording;
+  locale: en-US                  # replay keeps them
+  timezone: America/New_York
+```
+
+## Which rules apply
+
+A task is replayed with `sites/` and then `tasks/<id>/rules/` layered over each other:
+
+1. Hand-written files (`<site>.yaml`) win over inferred ones (`<site>.auto.yaml`), wherever they are.
+2. Among hand-written files, the task's own win over `sites/`.
+3. Endpoints of the winning file are tried first and its single values are kept; lists such as
+   `drop_params` and `block` are combined.
+
+So a correction that only matters for one task goes into `tasks/<id>/rules/<site>.yaml`, and one
+that helps every task on the site goes into `sites/<site>.yaml`. See [site-rules.md](site-rules.md).
+
+## How a task is added
+
+Recordings are made with the maintainers' recording tool, which is not part of this repository.
+The steps are:
+
+1. A maintainer writes `tasks/<id>/task.yaml`, or the person recording adds the task in the
+   recording tool, which hands the card over with the recording.
+2. A person records the task: browse first (searches in several wordings, product pages, policy
+   pages), then do once what the task asks of the cart, ending in the state it asks for (replay
+   answers cart pages with their last recording). Bot checks are completed by that
+   person, never by a script; orders are never placed. The tool packs the recording, removes
+   cookies, credentials and listed personal data, infers site rules, and replays every recorded
+   page once to measure hit rates.
+3. A maintainer imports the recording. The import refuses recordings that fail `pab audit`,
+   writes `recording/` and `rules/*.auto.yaml`, and leaves hand-written files
+   alone. Tasks are recorded signed out, so page text is not scanned for personal data.
+4. The maintainer replays the task (`pab replay <id>`), fixes misses with rules, and opens a pull
+   request.
+
+Recording a task again adds to it: the new recording is merged into `recording/`, requests
+recorded both times keep answering with the earlier copy, and the `*.auto.yaml` files are
+inferred again from both; cart pages answer with the new recording, the last one. The import can
+instead prefer the new copies or replace the recording.
+
+To propose a task, open an issue with the instruction and the sites involved.