personal-agent-benchmark 0.0.0-stage → 0.2.0

package/dist/src/core/archive.js

@@ -0,0 +1,213 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RECORDING_FILES = exports.RECORDING_FORMAT = exports.isBundleFile = exports.PUBLIC_BUNDLE_FILES = exports.PART_BYTES = void 0;
+exports.createArchive = createArchive;
+exports.archiveParts = archiveParts;
+exports.hashParts = hashParts;
+exports.extractArchive = extractArchive;
+exports.publishRecording = publishRecording;
+exports.recordingFiles = recordingFiles;
+exports.unpackRecording = unpackRecording;
+const node_crypto_1 = require("node:crypto");
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const node_stream_1 = require("node:stream");
+const promises_1 = require("node:stream/promises");
+const zlib = __importStar(require("node:zlib"));
+const tar = __importStar(require("tar"));
+const util_1 = require("./util");
+/**
+ * Large files travel as gzip-compressed tar files: the WARC of a published recording (below) and
+ * the recording tool's hand-off archive. Git hosts reject large files (GitHub: 100 MB, with a
+ * warning from 50 MB), so archives above PART_BYTES are split into numbered parts:
+ * warc.tar.gz.000, warc.tar.gz.001, ... which are read back in order as one stream.
+ */
+exports.PART_BYTES = 45 * 1024 * 1024;
+/** Files of a bundle that can be replayed and published; everything else stays with the recorder. */
+exports.PUBLIC_BUNDLE_FILES = ['manifest.json', 'archive/data.warc', 'indexes/entries.jsonl', 'pages/pages.jsonl'];
+/** Written by earlier versions and never read by replay; still accepted when unpacking old archives. */
+const LEGACY_BUNDLE_FILES = ['datapackage.json', 'indexes/index.cdxj'];
+const isBundleFile = (p) => exports.PUBLIC_BUNDLE_FILES.includes(p) || LEGACY_BUNDLE_FILES.includes(p);
+exports.isBundleFile = isBundleFile;
+// a fixed timestamp keeps archives of the same files byte-identical
+const ARCHIVE_MTIME = new Date('2000-01-01T00:00:00Z');
+/**
+ * Writes `files` (paths relative to `srcDir`, in this order) to `outFile`, a .tar.gz, replacing
+ * any earlier archive or parts there. Returns the files written: `outFile` or its parts.
+ */
+async function createArchive(srcDir, files, outFile, partBytes = exports.PART_BYTES) {
+    for (const f of files)
+        if (!fs.existsSync(path.join(srcDir, f)))
+            throw new Error(`missing ${f} in ${srcDir}`);
+    for (const old of archiveParts(outFile))
+        fs.rmSync(old);
+    fs.mkdirSync(path.dirname(outFile), { recursive: true });
+    const tmp = `${outFile}.tmp-${process.pid}`;
+    await tar.c({ file: tmp, cwd: srcDir, gzip: { level: 9 }, portable: true, mtime: ARCHIVE_MTIME }, files);
+    const size = fs.statSync(tmp).size;
+    if (size <= partBytes) {
+        fs.renameSync(tmp, outFile);
+        return [outFile];
+    }
+    const fd = fs.openSync(tmp, 'r');
+    const parts = [];
+    try {
+        const buf = Buffer.alloc(partBytes);
+        for (let offset = 0, n = 0; offset < size; offset += partBytes, n++) {
+            const read = fs.readSync(fd, buf, 0, partBytes, offset);
+            const part = `${outFile}.${String(n).padStart(3, '0')}`;
+            fs.writeFileSync(part, buf.subarray(0, read));
+            parts.push(part);
+        }
+    }
+    finally {
+        fs.closeSync(fd);
+        fs.rmSync(tmp);
+    }
+    return parts;
+}
+/** The archive at `file` as stored: the file itself, or its numbered parts in order. */
+function archiveParts(file) {
+    if (fs.existsSync(file))
+        return [file];
+    const dir = path.dirname(file);
+    if (!fs.existsSync(dir))
+        return [];
+    const base = path.basename(file);
+    return fs
+        .readdirSync(dir)
+        .filter((name) => name.startsWith(base + '.') && /^\d{3}$/.test(name.slice(base.length + 1)))
+        .sort()
+        .map((name) => path.join(dir, name));
+}
+function hashParts(parts) {
+    const hash = (0, node_crypto_1.createHash)('sha256');
+    for (const p of parts)
+        hash.update(fs.readFileSync(p));
+    return hash.digest('hex');
+}
+/**
+ * Extracts an archive (one file or its parts) into `destDir`. Only regular files and directories
+ * are extracted, and only paths accepted by `allow`; tar itself refuses absolute paths and "..".
+ */
+async function extractArchive(parts, destDir, allow = () => true) {
+    if (!parts.length)
+        throw new Error('no archive to extract');
+    fs.mkdirSync(destDir, { recursive: true });
+    const extracted = [];
+    const rejected = [];
+    const source = node_stream_1.Readable.from((async function* () {
+        for (const p of parts)
+            yield* fs.createReadStream(p);
+    })());
+    await (0, promises_1.pipeline)(source, tar.x({
+        cwd: destDir,
+        strict: true,
+        filter: (p, entry) => {
+            const type = 'type' in entry ? entry.type : 'File';
+            const name = p.replace(/^\.\//, '');
+            const ok = (type === 'File' || type === 'Directory') && !name.split('/').includes('..') && !path.isAbsolute(name) && (type === 'Directory' || allow(name));
+            if (ok && type === 'File')
+                extracted.push(name);
+            if (!ok)
+                rejected.push(name);
+            return ok;
+        },
+    }));
+    if (rejected.length)
+        throw new Error(`archive holds unexpected entries: ${rejected.slice(0, 5).join(', ')}`);
+    return extracted;
+}
+/**
+ * Layout version of a recording published in a task. Replay reads only this version: a change to
+ * the layout or to the files in it bumps the number, and existing tasks are rewritten to match.
+ */
+exports.RECORDING_FORMAT = 2;
+/**
+ * A bundle as published in a task (tasks/<id>/recording/):
+ *   manifest.json      the bundle's manifest plus "format"
+ *   pages.jsonl        pages/pages.jsonl
+ *   entries.jsonl.gz   indexes/entries.jsonl, gzip-compressed
+ *   warc.tar.gz        archive/data.warc (warc.tar.gz.000, .001, ... above PART_BYTES)
+ * The WARC sits apart from the files whose layout may change, so a new format rewrites only the
+ * small files and leaves the large archive as it was.
+ */
+exports.RECORDING_FILES = {
+    manifest: 'manifest.json',
+    pages: 'pages.jsonl',
+    entries: 'entries.jsonl.gz',
+    warc: 'warc.tar.gz',
+};
+/** Writes the bundle at `bundleDir` to `outDir` in the published layout, replacing what is there. */
+async function publishRecording(bundleDir, outDir) {
+    for (const f of exports.PUBLIC_BUNDLE_FILES)
+        if (!fs.existsSync(path.join(bundleDir, f)))
+            throw new Error(`missing ${f} in ${bundleDir}`);
+    fs.rmSync(outDir, { recursive: true, force: true });
+    const out = (name) => path.join(outDir, name);
+    (0, util_1.writeJson)(out(exports.RECORDING_FILES.manifest), { ...(0, util_1.readJson)(path.join(bundleDir, 'manifest.json')), format: exports.RECORDING_FORMAT });
+    fs.copyFileSync(path.join(bundleDir, 'pages', 'pages.jsonl'), out(exports.RECORDING_FILES.pages));
+    fs.writeFileSync(out(exports.RECORDING_FILES.entries), zlib.gzipSync(fs.readFileSync(path.join(bundleDir, 'indexes', 'entries.jsonl')), { level: 9 }));
+    const warc = await createArchive(bundleDir, ['archive/data.warc'], out(exports.RECORDING_FILES.warc));
+    return [out(exports.RECORDING_FILES.manifest), out(exports.RECORDING_FILES.pages), out(exports.RECORDING_FILES.entries), ...warc];
+}
+/** Files of the recording published in `dir`, in a fixed order; empty when there is none. */
+function recordingFiles(dir) {
+    const manifest = path.join(dir, exports.RECORDING_FILES.manifest);
+    if (!fs.existsSync(manifest))
+        return [];
+    return [manifest, path.join(dir, exports.RECORDING_FILES.pages), path.join(dir, exports.RECORDING_FILES.entries), ...archiveParts(path.join(dir, exports.RECORDING_FILES.warc))];
+}
+/** Turns the recording published in `dir` back into a bundle at `destDir`; refuses other formats. */
+async function unpackRecording(dir, destDir) {
+    const manifest = (0, util_1.readJson)(path.join(dir, exports.RECORDING_FILES.manifest));
+    if (manifest.format !== exports.RECORDING_FORMAT) {
+        throw new Error(`${dir} is recording format ${manifest.format ?? '(none)'}; this version reads format ${exports.RECORDING_FORMAT} (docs/bundle-format.md)`);
+    }
+    const warc = archiveParts(path.join(dir, exports.RECORDING_FILES.warc));
+    for (const f of [exports.RECORDING_FILES.pages, exports.RECORDING_FILES.entries])
+        if (!fs.existsSync(path.join(dir, f)))
+            throw new Error(`missing ${f} in ${dir}`);
+    if (!warc.length)
+        throw new Error(`missing ${exports.RECORDING_FILES.warc} in ${dir}`);
+    await extractArchive(warc, destDir, (p) => p === 'archive/data.warc');
+    fs.mkdirSync(path.join(destDir, 'pages'), { recursive: true });
+    fs.mkdirSync(path.join(destDir, 'indexes'), { recursive: true });
+    fs.copyFileSync(path.join(dir, exports.RECORDING_FILES.manifest), path.join(destDir, 'manifest.json'));
+    fs.copyFileSync(path.join(dir, exports.RECORDING_FILES.pages), path.join(destDir, 'pages', 'pages.jsonl'));
+    fs.writeFileSync(path.join(destDir, 'indexes', 'entries.jsonl'), zlib.gunzipSync(fs.readFileSync(path.join(dir, exports.RECORDING_FILES.entries))));
+}
+//# sourceMappingURL=archive.js.map

package/dist/src/core/archive.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"archive.js","sourceRoot":"","sources":["../../../src/core/archive.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAAyC;AACzC,MAAY,EAAE,oCAAgB;AAC9B,MAAY,IAAI,sCAAkB;AAClC,6CAAuC;AACvC,mDAAgD;AAChD,MAAY,IAAI,sCAAkB;AAClC,MAAY,GAAG,gCAAY;AAE3B,iCAA6C;AAE7C;;;;;GAKG;AACU,QAAA,UAAU,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAE3C,qGAAqG;AACxF,QAAA,mBAAmB,GAAG,CAAC,eAAe,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,mBAAmB,CAAC,CAAC;AAExH,wGAAwG;AACxG,MAAM,mBAAmB,GAAG,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;AAEhE,MAAM,YAAY,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,QAAA,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAAjG,QAAA,YAAY,GAAZ,YAAY,CAAqF;AAE9G,oEAAoE;AACpE,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,sBAAsB,CAAC,CAAC;AAEvD;;;GAGG;AACI,KAAK,wBAAwB,MAAc,EAAE,KAAe,EAAE,OAAe,EAAE,SAAS,GAAG,QAAA,UAAU;IAC1G,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,OAAO,MAAM,EAAE,CAAC,CAAC;IAC9G,KAAK,MAAM,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC;QAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACxD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,GAAG,GAAG,GAAG,OAAO,QAAQ,OAAO,CAAC,GAAG,EAAE,CAAC;IAC5C,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,KAAK,CAAC,CAAC;IACzG,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;IACnC,IAAI,IAAI,IAAI,SAAS,EAAE,CAAC;QACtB,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC5B,OAAO,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IACD,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACjC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACpC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACpE,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YACxD,MAAM,IAAI,GAAG,GAAG,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YACxD,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;YAC9C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACjB,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,wFAAwF;AACxF,sBAA6B,IAAY;IACvC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACjC,OAAO,EAAE;SACN,WAAW,CAAC,GAAG,CAAC;SAChB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;SAC5F,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,mBAA0B,KAAe;IACvC,MAAM,IAAI,GAAG,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACI,KAAK,yBAAyB,KAAe,EAAE,OAAe,EAAE,KAAK,GAA2B,GAAG,EAAE,CAAC,IAAI;IAC/G,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;IAC5D,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,sBAAQ,CAAC,IAAI,CAC1B,CAAC,KAAK,SAAS,CAAC;QACd,KAAK,MAAM,CAAC,IAAI,KAAK;YAAE,KAAK,CAAC,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC,CAAC,EAAE,CACL,CAAC;IACF,MAAM,IAAA,mBAAQ,EACZ,MAAM,EACN,GAAG,CAAC,CAAC,CAAC;QACJ,GAAG,EAAE,OAAO;QACZ,MAAM,EAAE,IAAI;QACZ,MAAM,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;YACnB,MAAM,IAAI,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;YACnD,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACpC,MAAM,EAAE,GAAG,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAC3J,IAAI,EAAE,IAAI,IAAI,KAAK,MAAM;gBAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChD,IAAI,CAAC,EAAE;gBAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,OAAO,EAAE,CAAC;QACZ,CAAC;KACF,CAAC,CACH,CAAC;IACF,IAAI,QAAQ,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7G,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;GAGG;AACU,QAAA,gBAAgB,GAAG,CAAC,CAAC;AAElC;;;;;;;;GAQG;AACU,QAAA,eAAe,GAAG;IAC7B,QAAQ,EAAE,eAAe;IACzB,KAAK,EAAE,aAAa;IACpB,OAAO,EAAE,kBAAkB;IAC3B,IAAI,EAAE,aAAa;CACX,CAAC;AAEX,qGAAqG;AAC9F,KAAK,2BAA2B,SAAiB,EAAE,MAAc;IACtE,KAAK,MAAM,CAAC,IAAI,QAAA,mBAAmB;QAAE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,OAAO,SAAS,EAAE,CAAC,CAAC;IAClI,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,MAAM,GAAG,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACtD,IAAA,gBAAS,EAAC,GAAG,CAAC,QAAA,eAAe,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,IAAA,eAAQ,EAAW,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC,EAAE,MAAM,EAAE,QAAA,gBAAgB,EAAE,CAAC,CAAC;IACrI,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,aAAa,CAAC,EAAE,GAAG,CAAC,QAAA,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1F,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,QAAA,eAAe,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/I,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,CAAC,mBAAmB,CAAC,EAAE,GAAG,CAAC,QAAA,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9F,OAAO,CAAC,GAAG,CAAC,QAAA,eAAe,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,QAAA,eAAe,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,QAAA,eAAe,CAAC,OAAO,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;AAC5G,CAAC;AAED,6FAA6F;AAC7F,wBAA+B,GAAW;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAA,eAAe,CAAC,QAAQ,CAAC,CAAC;IAC1D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,EAAE,CAAC;IACxC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAA,eAAe,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAA,eAAe,CAAC,OAAO,CAAC,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAA,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAC3J,CAAC;AAED,qGAAqG;AAC9F,KAAK,0BAA0B,GAAW,EAAE,OAAe;IAChE,MAAM,QAAQ,GAAG,IAAA,eAAQ,EAAW,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAA,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9E,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAA,gBAAgB,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,wBAAwB,QAAQ,CAAC,MAAM,IAAI,QAAQ,+BAA+B,QAAA,gBAAgB,0BAA0B,CAAC,CAAC;IACtJ,CAAC;IACD,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAA,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;IAChE,KAAK,MAAM,CAAC,IAAI,CAAC,QAAA,eAAe,CAAC,KAAK,EAAE,QAAA,eAAe,CAAC,OAAO,CAAC;QAAE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;IACnJ,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,QAAA,eAAe,CAAC,IAAI,OAAO,GAAG,EAAE,CAAC,CAAC;IAC/E,MAAM,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,mBAAmB,CAAC,CAAC;IACtE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAA,eAAe,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC;IAC/F,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAA,eAAe,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC;IACnG,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAA,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9I,CAAC"}

package/dist/src/core/audit.d.ts

@@ -0,0 +1,15 @@
+import type { Bundle } from './bundle';
+import type { RuleSet } from './rules';
+export interface AuditFinding {
+    check: string;
+    detail: string;
+}
+/**
+ * Checks that a bundle is safe to publish: no credentials in stored requests, no live cookie
+ * values in stored responses, no stored header lines that are not headers (a header sent several
+ * times stored as one value joined by newlines hides every cookie after the first from the cookie
+ * check), no pages the rules exclude (bot checks), and nothing in the manifest beyond what replay
+ * reads (such as who recorded it).
+ * Returns one finding per problem; an empty list means the bundle passed.
+ */
+export declare function auditBundle(bundle: Bundle, rules: RuleSet): Promise<AuditFinding[]>;

package/dist/src/core/audit.js

@@ -0,0 +1,85 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.auditBundle = auditBundle;
+const fs = __importStar(require("node:fs"));
+const warc_1 = require("./warc");
+const MANIFEST_KEYS = ['task_id', 'captured_from', 'captured_to', 'not_found', 'locale', 'timezone', 'format'];
+/** Written by recordings made before 2026-10-26; replay ignores it and the next import drops it. */
+const LEGACY_MANIFEST_KEYS = ['start_url'];
+const CREDENTIAL_REQUEST_HEADERS = ['cookie', 'authorization', 'proxy-authorization', 'x-amz-access-token'];
+/**
+ * Checks that a bundle is safe to publish: no credentials in stored requests, no live cookie
+ * values in stored responses, no stored header lines that are not headers (a header sent several
+ * times stored as one value joined by newlines hides every cookie after the first from the cookie
+ * check), no pages the rules exclude (bot checks), and nothing in the manifest beyond what replay
+ * reads (such as who recorded it).
+ * Returns one finding per problem; an empty list means the bundle passed.
+ */
+async function auditBundle(bundle, rules) {
+    const findings = [];
+    for (const f of [bundle.manifestPath, bundle.archivePath, bundle.entriesPath]) {
+        if (!fs.existsSync(f))
+            findings.push({ check: 'files', detail: `missing ${f}` });
+    }
+    if (findings.length)
+        return findings;
+    const extra = Object.keys(bundle.manifest()).filter((k) => !MANIFEST_KEYS.includes(k) && !LEGACY_MANIFEST_KEYS.includes(k));
+    if (extra.length)
+        findings.push({ check: 'manifest', detail: `manifest.json records fields replay does not use: ${extra.join(', ')}` });
+    const add = (check, detail) => {
+        if (findings.filter((f) => f.check === check).length < 20)
+            findings.push({ check, detail });
+    };
+    for await (const ex of (0, warc_1.readExchanges)(bundle.archivePath)) {
+        for (const [name] of ex.reqHeaders) {
+            if (CREDENTIAL_REQUEST_HEADERS.includes(name.toLowerCase()))
+                add('request-credentials', `${name} header on ${ex.method} ${ex.url}`);
+        }
+        if (ex.malformedHeaderLines)
+            add('header-lines', `${ex.malformedHeaderLines} stored header lines are not headers (values joined by newlines?) on ${ex.method} ${ex.url}`);
+        for (const [name, value] of ex.respHeaders) {
+            if (name.toLowerCase() !== 'set-cookie')
+                continue;
+            const cookieValue = value.split(';')[0].split('=').slice(1).join('=').trim();
+            if (cookieValue && cookieValue !== 'REDACTED')
+                add('response-cookies', `unredacted Set-Cookie on ${ex.url}`);
+        }
+        if (rules.isExcluded(ex.url))
+            add('excluded-pages', `excluded URL stored: ${ex.url}`);
+    }
+    return findings;
+}
+//# sourceMappingURL=audit.js.map

package/dist/src/core/audit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"audit.js","sourceRoot":"","sources":["../../../src/core/audit.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,MAAY,EAAE,oCAAgB;AAI9B,iCAAuC;AAOvC,MAAM,aAAa,GAAa,CAAC,SAAS,EAAE,eAAe,EAAE,aAAa,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,CAA8B,CAAC;AACtJ,oGAAoG;AACpG,MAAM,oBAAoB,GAAG,CAAC,WAAW,CAAC,CAAC;AAC3C,MAAM,0BAA0B,GAAG,CAAC,QAAQ,EAAE,eAAe,EAAE,qBAAqB,EAAE,oBAAoB,CAAC,CAAC;AAE5G;;;;;;;GAOG;AACI,KAAK,sBAAsB,MAAc,EAAE,KAAc;IAC9D,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9E,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM;QAAE,OAAO,QAAQ,CAAC;IAErC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5H,IAAI,KAAK,CAAC,MAAM;QAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,qDAAqD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAExI,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,MAAc,EAAE,EAAE;QAC5C,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,MAAM,GAAG,EAAE;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9F,CAAC,CAAC;IACF,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,IAAA,oBAAa,EAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QACzD,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,EAAE,CAAC;YACnC,IAAI,0BAA0B,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBAAE,GAAG,CAAC,qBAAqB,EAAE,GAAG,IAAI,cAAc,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QACtI,CAAC;QACD,IAAI,EAAE,CAAC,oBAAoB;YAAE,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,CAAC,oBAAoB,wEAAwE,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1K,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC3C,IAAI,IAAI,CAAC,WAAW,EAAE,KAAK,YAAY;gBAAE,SAAS;YAClD,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7E,IAAI,WAAW,IAAI,WAAW,KAAK,UAAU;gBAAE,GAAG,CAAC,kBAAkB,EAAE,4BAA4B,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAC/G,CAAC;QACD,IAAI,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,gBAAgB,EAAE,wBAAwB,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;IACxF,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/src/core/bundle.d.ts

@@ -0,0 +1,42 @@
+import type { Entry, Manifest } from './types';
+/**
+ * A recording bundle on disk. One recording, one directory:
+ *   manifest.json, archive/data.warc, indexes/entries.jsonl, pages/pages.jsonl;
+ *   optionally rules/*.yaml.
+ * The recorder also keeps working files next to them (raw/, review/, check/, report.json);
+ * those never leave the recording machine.
+ */
+export declare class Bundle {
+    readonly dir: string;
+    constructor(dir: string);
+    get manifestPath(): string;
+    /** archive/data.warc; bundles written before the switch to uncompressed records use data.warc.gz. */
+    get archivePath(): string;
+    get entriesPath(): string;
+    get pagesPath(): string;
+    get rulesDir(): string;
+    get reviewDir(): string;
+    get rawDir(): string;
+    get checkDir(): string;
+    get reportPath(): string;
+    isPacked(): boolean;
+    manifest(): Manifest;
+    entries(): Entry[];
+    pages(): PageRecord[];
+}
+/** A top-level page visited during capture (WACZ pages.jsonl shape plus capture notes). */
+export interface PageRecord {
+    id: string;
+    url: string;
+    ts: string;
+    title?: string;
+    /** e.g. "not-found-probe", "reload", "after-add-to-cart" */
+    note?: string;
+    /** Saved accessibility snapshot / screenshot names under raw/snapshots. */
+    snapshot?: string;
+    /**
+     * How the page's URL was fetched during capture: "GET" (a real document load), "POST" (form
+     * result), or "none" (the address bar was rewritten by page script, nothing was requested).
+     */
+    fetched?: 'GET' | 'POST' | 'none';
+}

package/dist/src/core/bundle.js

@@ -0,0 +1,80 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Bundle = void 0;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const util_1 = require("./util");
+/**
+ * A recording bundle on disk. One recording, one directory:
+ *   manifest.json, archive/data.warc, indexes/entries.jsonl, pages/pages.jsonl;
+ *   optionally rules/*.yaml.
+ * The recorder also keeps working files next to them (raw/, review/, check/, report.json);
+ * those never leave the recording machine.
+ */
+class Bundle {
+    dir;
+    constructor(dir) {
+        this.dir = dir;
+    }
+    get manifestPath() { return path.join(this.dir, 'manifest.json'); }
+    /** archive/data.warc; bundles written before the switch to uncompressed records use data.warc.gz. */
+    get archivePath() {
+        const plain = path.join(this.dir, 'archive', 'data.warc');
+        const gz = plain + '.gz';
+        return !fs.existsSync(plain) && fs.existsSync(gz) ? gz : plain;
+    }
+    get entriesPath() { return path.join(this.dir, 'indexes', 'entries.jsonl'); }
+    get pagesPath() { return path.join(this.dir, 'pages', 'pages.jsonl'); }
+    get rulesDir() { return path.join(this.dir, 'rules'); }
+    get reviewDir() { return path.join(this.dir, 'review'); }
+    get rawDir() { return path.join(this.dir, 'raw'); }
+    get checkDir() { return path.join(this.dir, 'check'); }
+    get reportPath() { return path.join(this.dir, 'report.json'); }
+    isPacked() {
+        return fs.existsSync(this.manifestPath) && fs.existsSync(this.entriesPath) && fs.existsSync(this.archivePath);
+    }
+    manifest() {
+        return (0, util_1.readJson)(this.manifestPath);
+    }
+    entries() {
+        return (0, util_1.readJsonl)(this.entriesPath);
+    }
+    pages() {
+        return (0, util_1.readJsonl)(this.pagesPath);
+    }
+}
+exports.Bundle = Bundle;
+//# sourceMappingURL=bundle.js.map

package/dist/src/core/bundle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bundle.js","sourceRoot":"","sources":["../../../src/core/bundle.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,MAAY,EAAE,oCAAgB;AAC9B,MAAY,IAAI,sCAAkB;AAElC,iCAA6C;AAE7C;;;;;;GAMG;AACH;IACuB,GAAG;IAAxB,YAAqB,GAAW;mBAAX,GAAG;IAAW,CAAC;IAEpC,IAAI,YAAY,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;IAC3E,qGAAqG;IACrG,IAAI,WAAW;QACb,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;QAC1D,MAAM,EAAE,GAAG,KAAK,GAAG,KAAK,CAAC;QACzB,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IACjE,CAAC;IACD,IAAI,WAAW,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;IACrF,IAAI,SAAS,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;IAC/E,IAAI,QAAQ,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/D,IAAI,SAAS,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IACjE,IAAI,MAAM,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC3D,IAAI,QAAQ,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/D,IAAI,UAAU,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;IAEvE,QAAQ;QACN,OAAO,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAChH,CAAC;IAED,QAAQ;QACN,OAAO,IAAA,eAAQ,EAAW,IAAI,CAAC,YAAY,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO;QACL,OAAO,IAAA,gBAAS,EAAQ,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK;QACH,OAAO,IAAA,gBAAS,EAAa,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC;CACF"}

package/dist/src/core/lookup.d.ts

@@ -0,0 +1,15 @@
+import { type NormalizedRequest } from './normalize';
+import type { IndexedEntry, RecordStore } from './store';
+export interface LookupResult {
+    source: 'exact' | 'fuzzy' | 'search' | 'promo';
+    ix: IndexedEntry;
+    detail?: string;
+}
+/**
+ * The lookup ladder: exact key, then free-text (search / promo), then same-path fuzzy.
+ * When a key was recorded several times, the earliest recording wins ("browse first, act later"),
+ * except for pages that follow what was done (carts, `latest` in the rules): there the last does.
+ */
+export declare function lookup(store: RecordStore, norm: NormalizedRequest): LookupResult | undefined;
+/** Share of parameters (by name) whose values agree; 1 when neither side has parameters. */
+export declare function paramOverlap(a: [string, string][], b: [string, string][]): number;

package/dist/src/core/lookup.js

@@ -0,0 +1,82 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.lookup = lookup;
+exports.paramOverlap = paramOverlap;
+const normalize_1 = require("./normalize");
+const util_1 = require("./util");
+const DEFAULT_SIMILARITY = 0.6;
+const DEFAULT_FUZZY_MIN = 0.5;
+/**
+ * The lookup ladder: exact key, then free-text (search / promo), then same-path fuzzy.
+ * When a key was recorded several times, the earliest recording wins ("browse first, act later"),
+ * except for pages that follow what was done (carts, `latest` in the rules): there the last does.
+ */
+function lookup(store, norm) {
+    const exact = store.byKey.get(norm.key);
+    if (exact && exact.length)
+        return { source: 'exact', ix: norm.latest ? exact[exact.length - 1] : exact[0] };
+    const sameEndpoint = (store.byPathKey.get(norm.pathKey) ?? []).filter((ix) => ix.entry.status !== 0);
+    if (norm.kind === 'search' && norm.text !== undefined) {
+        const found = bestSearch(sameEndpoint, norm);
+        if (found)
+            return found;
+    }
+    if (norm.kind === 'promo') {
+        const code = norm.text ?? '';
+        const same = sameEndpoint.find((ix) => ix.norm.text === code);
+        if (same)
+            return { source: 'promo', ix: same, detail: 'known-code' };
+        const invalid = norm.endpoint?.invalid_code ? (0, normalize_1.normalizeText)(norm.endpoint.invalid_code) : undefined;
+        const fallback = invalid !== undefined ? sameEndpoint.find((ix) => ix.norm.text === invalid) : undefined;
+        if (fallback)
+            return { source: 'promo', ix: fallback, detail: 'unknown-code' };
+        return undefined;
+    }
+    const fuzzyAllowed = norm.endpoint?.fuzzy ?? norm.bodyKey === '';
+    if (!fuzzyAllowed || sameEndpoint.length === 0)
+        return undefined;
+    const min = norm.endpoint?.fuzzy_min ?? (norm.kind === 'static' ? 0 : DEFAULT_FUZZY_MIN);
+    let best;
+    // among equally good matches the first one tried wins
+    for (const ix of norm.latest ? [...sameEndpoint].reverse() : sameEndpoint) {
+        const score = paramOverlap(norm.params, ix.norm.params);
+        if (score < min)
+            continue;
+        if (!best || score > best.score)
+            best = { ix, score };
+    }
+    return best ? { source: 'fuzzy', ix: best.ix, detail: `overlap=${best.score.toFixed(2)}` } : undefined;
+}
+function bestSearch(candidates, norm) {
+    const want = (0, util_1.tokens)(norm.text ?? '');
+    const textParam = norm.endpoint?.text_param;
+    const others = norm.params.filter(([k]) => k !== textParam);
+    const threshold = norm.endpoint?.similarity ?? DEFAULT_SIMILARITY;
+    let best;
+    for (const ix of candidates) {
+        if (ix.norm.text === undefined)
+            continue;
+        const sim = (0, util_1.jaccard)(want, (0, util_1.tokens)(ix.norm.text));
+        if (sim < threshold)
+            continue;
+        const rest = paramOverlap(others, ix.norm.params.filter(([k]) => k !== textParam));
+        const score = sim + rest * 0.01;
+        if (!best || score > best.score)
+            best = { ix, score };
+    }
+    return best ? { source: 'search', ix: best.ix, detail: `similarity=${best.score.toFixed(2)}` } : undefined;
+}
+/** Share of parameters (by name) whose values agree; 1 when neither side has parameters. */
+function paramOverlap(a, b) {
+    const ma = new Map(a);
+    const mb = new Map(b);
+    const names = new Set([...ma.keys(), ...mb.keys()]);
+    if (names.size === 0)
+        return 1;
+    let same = 0;
+    for (const n of names)
+        if (ma.has(n) && ma.get(n) === mb.get(n))
+            same++;
+    return same / names.size;
+}
+//# sourceMappingURL=lookup.js.map

package/dist/src/core/lookup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lookup.js","sourceRoot":"","sources":["../../../src/core/lookup.ts"],"names":[],"mappings":";;;;AAAA,2CAAoE;AAEpE,iCAAyC;AAQzC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B;;;;GAIG;AACH,gBAAuB,KAAkB,EAAE,IAAuB;IAChE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAE5G,MAAM,YAAY,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;IAErG,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,UAAU,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAC7C,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;IAC1B,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QAC9D,IAAI,IAAI;YAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;QACrE,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,IAAA,yBAAa,EAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACpG,MAAM,QAAQ,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACzG,IAAI,QAAQ;YAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;QAC/E,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,IAAI,IAAI,CAAC,OAAO,KAAK,EAAE,CAAC;IACjE,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACjE,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;IACzF,IAAI,IAAqD,CAAC;IAC1D,sDAAsD;IACtD,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC;QAC1E,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,KAAK,GAAG,GAAG;YAAE,SAAS;QAC1B,IAAI,CAAC,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK;YAAE,IAAI,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;IACxD,CAAC;IACD,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,WAAW,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACzG,CAAC;AAED,SAAS,UAAU,CAAC,UAA0B,EAAE,IAAuB;IACrE,MAAM,IAAI,GAAG,IAAA,aAAM,EAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,IAAI,kBAAkB,CAAC;IAClE,IAAI,IAAqD,CAAC;IAC1D,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;QAC5B,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS;YAAE,SAAS;QACzC,MAAM,GAAG,GAAG,IAAA,cAAO,EAAC,IAAI,EAAE,IAAA,aAAM,EAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAChD,IAAI,GAAG,GAAG,SAAS;YAAE,SAAS;QAC9B,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC;QACnF,MAAM,KAAK,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK;YAAE,IAAI,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;IACxD,CAAC;IACD,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,cAAc,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7G,CAAC;AAED,4FAA4F;AAC5F,sBAA6B,CAAqB,EAAE,CAAqB;IACvE,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;IACtB,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;IACtB,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACpD,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,IAAI,EAAE,CAAC;IACxE,OAAO,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;AAC3B,CAAC"}

package/dist/src/core/normalize.d.ts

@@ -0,0 +1,45 @@
+import type { EndpointRule, Kind, SiteRules } from './types';
+import { RuleSet } from './rules';
+export interface RequestInput {
+    method: string;
+    url: string;
+    contentType?: string;
+    body?: Buffer;
+}
+export interface NormalizedRequest {
+    method: string;
+    host: string;
+    /** Path after applying the endpoint key template. */
+    path: string;
+    /** Kept query parameters, sorted. */
+    params: [string, string][];
+    bodyKey: string;
+    /** METHOD host/path?query #body:... — the exact-match key. */
+    key: string;
+    /** METHOD host/path — groups requests for fuzzy matching. */
+    pathKey: string;
+    kind: Kind;
+    site?: SiteRules;
+    endpoint?: EndpointRule;
+    captures: Record<string, string>;
+    /** Normalized free text for search and promo endpoints. */
+    text?: string;
+    /** Answer with the last recording of this request rather than the first (carts). */
+    latest: boolean;
+}
+export declare function normalizeText(value: string): string;
+export declare function normalizeRequest(input: RequestInput, rules: RuleSet): NormalizedRequest;
+type ParsedBody = {
+    kind: 'none';
+} | {
+    kind: 'fields';
+    fields: Record<string, unknown>;
+} | {
+    kind: 'raw';
+    raw: Buffer;
+};
+export declare function parseBody(contentType: string | undefined, body: Buffer | undefined): ParsedBody;
+export declare function getPath(obj: unknown, dotted: string): unknown;
+/** Read a field from the query string or a parsed body. */
+export declare function readField(input: RequestInput, from: 'query' | 'body', field: string): string | undefined;
+export {};