personal-agent-benchmark 0.0.0-stage → 0.2.0

package/dist/src/core/normalize.js

@@ -0,0 +1,143 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeText = normalizeText;
+exports.normalizeRequest = normalizeRequest;
+exports.parseBody = parseBody;
+exports.getPath = getPath;
+exports.readField = readField;
+const rules_1 = require("./rules");
+const util_1 = require("./util");
+const STATIC_EXT = /\.(js|mjs|css|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|mp4|webm|mp3|wav|map)$/i;
+function normalizeText(value) {
+    return value.replace(/\+/g, ' ').toLowerCase().replace(/\s+/g, ' ').trim();
+}
+function normalizeRequest(input, rules) {
+    const method = input.method.toUpperCase();
+    const url = new URL(input.url);
+    const host = url.hostname.toLowerCase() + (url.port && !isDefaultPort(url) ? ':' + url.port : '');
+    let pathname = url.pathname.replace(/\/{2,}/g, '/');
+    if (pathname.length > 1 && pathname.endsWith('/'))
+        pathname = pathname.slice(0, -1);
+    const site = rules.siteFor(url.hostname.toLowerCase());
+    const match = rules.matchEndpoint(site, method, url.hostname.toLowerCase(), pathname);
+    const endpoint = match?.endpoint;
+    const captures = match?.captures ?? {};
+    const path = endpoint?.key ? (0, rules_1.fillTemplate)(endpoint.key, captures) : pathname;
+    let kind = endpoint?.kind ?? (STATIC_EXT.test(pathname) ? 'static' : 'browse');
+    if (rules.isBlocked(url.hostname.toLowerCase(), pathname, site))
+        kind = 'block';
+    const textParam = endpoint?.text_param;
+    let text;
+    const drop = rules.dropParams(site, endpoint);
+    const params = [];
+    for (const [k, v] of url.searchParams) {
+        if (endpoint?.keep_params ? !endpoint.keep_params.includes(k) : (0, util_1.matchesAny)(k, drop))
+            continue;
+        if (textParam && k === textParam) {
+            text = normalizeText(v);
+            params.push([k, text]);
+        }
+        else {
+            params.push([k, v]);
+        }
+    }
+    params.sort((a, b) => (a[0] === b[0] ? a[1].localeCompare(b[1]) : a[0].localeCompare(b[0])));
+    const body = parseBody(input.contentType, input.body);
+    let bodyKey = '';
+    if (body.kind === 'fields') {
+        const kept = filterFields(body.fields, endpoint?.body_keep, rules.bodyDrop(site, endpoint));
+        if (textParam && typeof kept[textParam] === 'string') {
+            text = normalizeText(kept[textParam]);
+            kept[textParam] = text;
+        }
+        bodyKey = Object.keys(kept).length ? (0, util_1.stableStringify)(kept) : '';
+    }
+    else if (body.kind === 'raw') {
+        bodyKey = 'sha256:' + (0, util_1.sha256)(body.raw).slice(0, 16);
+    }
+    const query = params.map(([k, v]) => `${k}=${v}`).join('&');
+    const pathKey = `${method} ${host}${path}`;
+    const key = pathKey + (query ? '?' + query : '') + (bodyKey ? ' #body:' + bodyKey : '');
+    const latest = rules.prefersLatest(url.hostname.toLowerCase(), pathname, site);
+    return { method, host, path, params, bodyKey, key, pathKey, kind, site, endpoint, captures, text, latest };
+}
+function isDefaultPort(url) {
+    return (url.protocol === 'https:' && url.port === '443') || (url.protocol === 'http:' && url.port === '80');
+}
+function parseBody(contentType, body) {
+    if (!body || body.length === 0)
+        return { kind: 'none' };
+    const ct = (contentType ?? '').toLowerCase();
+    const text = body.toString('utf8');
+    if (ct.includes('json') || /^\s*[{[]/.test(text)) {
+        try {
+            const parsed = JSON.parse(text);
+            if (parsed && typeof parsed === 'object' && !Array.isArray(parsed))
+                return { kind: 'fields', fields: parsed };
+            return { kind: 'fields', fields: { _: parsed } };
+        }
+        catch {
+            // not JSON after all
+        }
+    }
+    if (ct.includes('x-www-form-urlencoded') || (!ct && /^[\w.%[\]-]+=/.test(text))) {
+        const fields = {};
+        for (const [k, v] of new URLSearchParams(text)) {
+            const prev = fields[k];
+            fields[k] = prev === undefined ? v : Array.isArray(prev) ? [...prev, v] : [prev, v];
+        }
+        return { kind: 'fields', fields };
+    }
+    return { kind: 'raw', raw: body };
+}
+/** Keep listed top-level fields (dot paths allowed) or drop volatile ones recursively. */
+function filterFields(fields, keep, drop) {
+    if (keep && keep.length) {
+        const out = {};
+        for (const p of keep) {
+            const v = getPath(fields, p);
+            if (v !== undefined)
+                out[p] = v;
+        }
+        return out;
+    }
+    const strip = (value) => {
+        if (Array.isArray(value))
+            return value.map(strip);
+        if (value && typeof value === 'object') {
+            const out = {};
+            for (const [k, v] of Object.entries(value)) {
+                if (!(0, util_1.matchesAny)(k, drop))
+                    out[k] = strip(v);
+            }
+            return out;
+        }
+        return value;
+    };
+    return strip(fields);
+}
+function getPath(obj, dotted) {
+    if (obj && typeof obj === 'object' && dotted in obj) {
+        return obj[dotted];
+    }
+    let cur = obj;
+    for (const part of dotted.split('.')) {
+        if (cur === null || typeof cur !== 'object')
+            return undefined;
+        cur = cur[part];
+    }
+    return cur;
+}
+/** Read a field from the query string or a parsed body. */
+function readField(input, from, field) {
+    if (from === 'query')
+        return new URL(input.url).searchParams.get(field) ?? undefined;
+    const body = parseBody(input.contentType, input.body);
+    if (body.kind !== 'fields')
+        return undefined;
+    const v = getPath(body.fields, field);
+    if (Array.isArray(v))
+        return v.length ? String(v[0]) : undefined;
+    return v === undefined || v === null ? undefined : String(v);
+}
+//# sourceMappingURL=normalize.js.map

package/dist/src/core/normalize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize.js","sourceRoot":"","sources":["../../../src/core/normalize.ts"],"names":[],"mappings":";;;;;;;AACA,mCAAgD;AAChD,iCAA6D;AA+B7D,MAAM,UAAU,GAAG,0FAA0F,CAAC;AAE9G,uBAA8B,KAAa;IACzC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC7E,CAAC;AAED,0BAAiC,KAAmB,EAAE,KAAc;IAClE,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;IAC1C,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAClG,IAAI,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACpD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAEpF,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,QAAQ,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,KAAK,EAAE,QAAQ,CAAC;IACjC,MAAM,QAAQ,GAAG,KAAK,EAAE,QAAQ,IAAI,EAAE,CAAC;IACvC,MAAM,IAAI,GAAG,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,IAAA,oBAAY,EAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAE7E,IAAI,IAAI,GAAS,QAAQ,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACrF,IAAI,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC;QAAE,IAAI,GAAG,OAAO,CAAC;IAEhF,MAAM,SAAS,GAAG,QAAQ,EAAE,UAAU,CAAC;IACvC,IAAI,IAAwB,CAAC;IAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAuB,EAAE,CAAC;IACtC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;QACtC,IAAI,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAA,iBAAU,EAAC,CAAC,EAAE,IAAI,CAAC;YAAE,SAAS;QAC9F,IAAI,SAAS,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7F,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IACtD,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC5F,IAAI,SAAS,IAAI,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,QAAQ,EAAE,CAAC;YACrD,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,SAAS,CAAW,CAAC,CAAC;YAChD,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;QACzB,CAAC;QACD,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAA,sBAAe,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,CAAC;SAAM,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC/B,OAAO,GAAG,SAAS,GAAG,IAAA,aAAM,EAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,GAAG,MAAM,IAAI,IAAI,GAAG,IAAI,EAAE,CAAC;IAC3C,MAAM,GAAG,GAAG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxF,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC/E,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC7G,CAAC;AAED,SAAS,aAAa,CAAC,GAAQ;IAC7B,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AAC9G,CAAC;AAID,mBAA0B,WAA+B,EAAE,IAAwB;IACjF,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IACxD,MAAM,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;YAC9G,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;QACnD,CAAC;QAAC,MAAM,CAAC;YACP,qBAAqB;QACvB,CAAC;IACH,CAAC;IACD,IAAI,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAChF,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACtF,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACpC,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;AACpC,CAAC;AAED,0FAA0F;AAC1F,SAAS,YAAY,CAAC,MAA+B,EAAE,IAA0B,EAAE,IAAc;IAC/F,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC7B,IAAI,CAAC,KAAK,SAAS;gBAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,KAAc,EAAW,EAAE;QACxC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAClD,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvC,MAAM,GAAG,GAA4B,EAAE,CAAC;YACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;gBACtE,IAAI,CAAC,IAAA,iBAAU,EAAC,CAAC,EAAE,IAAI,CAAC;oBAAE,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IACF,OAAO,KAAK,CAAC,MAAM,CAA4B,CAAC;AAClD,CAAC;AAED,iBAAwB,GAAY,EAAE,MAAc;IAClD,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,MAAM,IAAK,GAA+B,EAAE,CAAC;QACjF,OAAQ,GAA+B,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,GAAG,GAAY,GAAG,CAAC;IACvB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACrC,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAC;QAC9D,GAAG,GAAI,GAA+B,CAAC,IAAI,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,2DAA2D;AAC3D,mBAA0B,KAAmB,EAAE,IAAsB,EAAE,KAAa;IAClF,IAAI,IAAI,KAAK,OAAO;QAAE,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC;IACrF,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IACtD,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC7C,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACjE,OAAO,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC"}

package/dist/src/core/rules.d.ts

@@ -0,0 +1,38 @@
+import type { EndpointRule, SiteRules } from './types';
+export interface EndpointMatch {
+    endpoint: EndpointRule;
+    captures: Record<string, string>;
+}
+/** Site rules for one bundle: `_default.yaml` plus one file per site. */
+export declare class RuleSet {
+    readonly defaults: SiteRules;
+    readonly sites: SiteRules[];
+    constructor(sites: SiteRules[]);
+    /**
+     * Files for the same site are merged. Hand-written files (`<site>.yaml`) take precedence over
+     * generated ones (`<site>.auto.yaml`) from any directory; among files of the same kind, later
+     * directories win. Winning files' endpoints are tried first and their scalar settings kept;
+     * lists are combined.
+     */
+    static load(dirs: string[]): RuleSet;
+    siteFor(host: string): SiteRules | undefined;
+    dropParams(site?: SiteRules, endpoint?: EndpointRule): string[];
+    bodyDrop(site?: SiteRules, endpoint?: EndpointRule): string[];
+    isBlocked(host: string, pathname: string, site?: SiteRules): boolean;
+    /** True for URLs that must never be stored or replayed, such as bot-check pages. */
+    isExcluded(url: string): boolean;
+    /** True for pages answered with their last recording (see SiteRules.latest). */
+    prefersLatest(host: string, pathname: string, site?: SiteRules): boolean;
+    matchEndpoint(site: SiteRules | undefined, method: string, host: string, pathname: string): EndpointMatch | undefined;
+    allEndpoints(): {
+        site: SiteRules;
+        endpoint: EndpointRule;
+    }[];
+}
+export declare function parseMatch(match: string): {
+    method?: string;
+    pattern: string;
+};
+/** Match a path against a pattern; returns captured `{name}` segments or undefined. */
+export declare function matchPath(pattern: string, pathname: string): Record<string, string> | undefined;
+export declare function fillTemplate(template: string, captures: Record<string, string>): string;

package/dist/src/core/rules.js

@@ -0,0 +1,187 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RuleSet = void 0;
+exports.parseMatch = parseMatch;
+exports.matchPath = matchPath;
+exports.fillTemplate = fillTemplate;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const yaml_1 = __importDefault(require("yaml"));
+const util_1 = require("./util");
+const DEFAULT_SITE = '_default';
+/** Site rules for one bundle: `_default.yaml` plus one file per site. */
+class RuleSet {
+    defaults;
+    sites;
+    constructor(sites) {
+        this.defaults = sites.find((s) => s.site === DEFAULT_SITE) ?? { site: DEFAULT_SITE, hosts: [] };
+        this.sites = sites.filter((s) => s.site !== DEFAULT_SITE);
+    }
+    /**
+     * Files for the same site are merged. Hand-written files (`<site>.yaml`) take precedence over
+     * generated ones (`<site>.auto.yaml`) from any directory; among files of the same kind, later
+     * directories win. Winning files' endpoints are tried first and their scalar settings kept;
+     * lists are combined.
+     */
+    static load(dirs) {
+        const files = [];
+        dirs.forEach((dir, d) => {
+            if (!dir || !fs.existsSync(dir))
+                return;
+            for (const name of fs.readdirSync(dir).sort()) {
+                if (!/\.ya?ml$/.test(name) || name.includes('.draft.'))
+                    continue;
+                files.push({ file: path.join(dir, name), rank: (/\.auto\.ya?ml$/.test(name) ? 0 : dirs.length) + d });
+            }
+        });
+        files.sort((a, b) => b.rank - a.rank);
+        const bySite = new Map();
+        for (const { file } of files) {
+            const parsed = yaml_1.default.parse(fs.readFileSync(file, 'utf8'));
+            if (!parsed || !parsed.site)
+                throw new Error(`rules file without "site": ${file}`);
+            parsed.hosts = parsed.hosts ?? [];
+            const prev = bySite.get(parsed.site);
+            bySite.set(parsed.site, prev ? mergeSite(prev, parsed) : parsed);
+        }
+        return new RuleSet([...bySite.values()]);
+    }
+    siteFor(host) {
+        return this.sites.find((s) => (0, util_1.matchesAny)(host, s.hosts));
+    }
+    dropParams(site, endpoint) {
+        return [...(this.defaults.drop_params ?? []), ...(site?.drop_params ?? []), ...(endpoint?.drop_params ?? [])];
+    }
+    bodyDrop(site, endpoint) {
+        return [...(this.defaults.body_drop ?? []), ...(site?.body_drop ?? []), ...(endpoint?.body_drop ?? [])];
+    }
+    isBlocked(host, pathname, site) {
+        const patterns = [...(this.defaults.block ?? []), ...(site?.block ?? [])];
+        return patterns.some((p) => (p.includes('/') ? (0, util_1.globToRegExp)(p).test(host + pathname) : (0, util_1.globToRegExp)(p).test(host)));
+    }
+    /** True for URLs that must never be stored or replayed, such as bot-check pages. */
+    isExcluded(url) {
+        let u;
+        try {
+            u = new URL(url);
+        }
+        catch {
+            return false;
+        }
+        const host = u.hostname.toLowerCase();
+        const patterns = [...(this.defaults.exclude ?? []), ...(this.siteFor(host)?.exclude ?? [])];
+        return patterns.some((p) => (p.includes('/') ? (0, util_1.globToRegExp)(p).test(host + u.pathname) : (0, util_1.globToRegExp)(p).test(host)));
+    }
+    /** True for pages answered with their last recording (see SiteRules.latest). */
+    prefersLatest(host, pathname, site) {
+        const patterns = [...(this.defaults.latest ?? []), ...(site?.latest ?? [])];
+        return patterns.some((p) => (p.includes('/') ? (0, util_1.globToRegExp)(p).test(host + pathname) : (0, util_1.globToRegExp)(p).test(host)));
+    }
+    matchEndpoint(site, method, host, pathname) {
+        for (const endpoint of site?.endpoints ?? []) {
+            if (endpoint.host && !(0, util_1.globToRegExp)(endpoint.host).test(host))
+                continue;
+            const { method: m, pattern } = parseMatch(endpoint.match);
+            if (m && m !== method)
+                continue;
+            const captures = matchPath(pattern, pathname);
+            if (captures)
+                return { endpoint, captures };
+        }
+        return undefined;
+    }
+    allEndpoints() {
+        return this.sites.flatMap((site) => (site.endpoints ?? []).map((endpoint) => ({ site, endpoint })));
+    }
+}
+exports.RuleSet = RuleSet;
+/** `high` wins: its endpoints are tried first and its single values are kept; lists are unioned. */
+function mergeSite(high, low) {
+    const union = (a, b) => (a || b ? [...new Set([...(a ?? []), ...(b ?? [])])] : undefined);
+    const names = new Set((high.endpoints ?? []).map((e) => e.name));
+    return {
+        ...low,
+        ...high,
+        hosts: union(high.hosts, low.hosts) ?? [],
+        drop_params: union(high.drop_params, low.drop_params),
+        body_drop: union(high.body_drop, low.body_drop),
+        block: union(high.block, low.block),
+        exclude: union(high.exclude, low.exclude),
+        latest: union(high.latest, low.latest),
+        endpoints: [...(high.endpoints ?? []), ...(low.endpoints ?? []).filter((e) => !names.has(e.name))],
+    };
+}
+function parseMatch(match) {
+    const parts = match.trim().split(/\s+/);
+    if (parts.length === 2)
+        return { method: parts[0].toUpperCase(), pattern: parts[1] };
+    return { pattern: parts[0] };
+}
+/** Match a path against a pattern; returns captured `{name}` segments or undefined. */
+function matchPath(pattern, pathname) {
+    const pat = pattern.split('/').filter(Boolean);
+    const segs = pathname.split('/').filter(Boolean);
+    const captures = {};
+    const walk = (i, j) => {
+        if (i === pat.length)
+            return j === segs.length;
+        const p = pat[i];
+        if (p === '**') {
+            for (let k = j; k <= segs.length; k++)
+                if (walk(i + 1, k))
+                    return true;
+            return false;
+        }
+        if (j >= segs.length)
+            return false;
+        const capture = /^\{(\w+)\}$/.exec(p);
+        if (capture) {
+            captures[capture[1]] = decodeURIComponent(segs[j]);
+            return walk(i + 1, j + 1);
+        }
+        if (p === '*' || (0, util_1.globToRegExp)(p).test(segs[j]))
+            return walk(i + 1, j + 1);
+        return false;
+    };
+    return walk(0, 0) ? captures : undefined;
+}
+function fillTemplate(template, captures) {
+    return template.replace(/\{(\w+)\}/g, (_, name) => captures[name] ?? '');
+}
+//# sourceMappingURL=rules.js.map

package/dist/src/core/rules.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rules.js","sourceRoot":"","sources":["../../../src/core/rules.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,MAAY,EAAE,oCAAgB;AAC9B,MAAY,IAAI,sCAAkB;AAClC,gDAAwB;AAExB,iCAAkD;AAElD,MAAM,YAAY,GAAG,UAAU,CAAC;AAOhC,yEAAyE;AACzE;IACW,QAAQ,CAAY;IACpB,KAAK,CAAc;IAE5B,YAAY,KAAkB;QAC5B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QAChG,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,IAAc;QACxB,MAAM,KAAK,GAAqC,EAAE,CAAC;QACnD,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YACtB,IAAI,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,OAAO;YACxC,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC9C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;oBAAE,SAAS;gBACjE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxG,CAAC;QACH,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqB,CAAC;QAC5C,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,cAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAc,CAAC;YACtE,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,EAAE,CAAC,CAAC;YACnF,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,IAAI,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,CAAC,IAAY;QAClB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,iBAAU,EAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,UAAU,CAAC,IAAgB,EAAE,QAAuB;QAClD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC;IAChH,CAAC;IAED,QAAQ,CAAC,IAAgB,EAAE,QAAuB;QAChD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,SAAS,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,EAAE,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;IAC1G,CAAC;IAED,SAAS,CAAC,IAAY,EAAE,QAAgB,EAAE,IAAgB;QACxD,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1E,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAA,mBAAY,EAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAA,mBAAY,EAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtH,CAAC;IAED,oFAAoF;IACpF,UAAU,CAAC,GAAW;QACpB,IAAI,CAAM,CAAC;QACX,IAAI,CAAC;YACH,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;QAC5F,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAA,mBAAY,EAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAA,mBAAY,EAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxH,CAAC;IAED,gFAAgF;IAChF,aAAa,CAAC,IAAY,EAAE,QAAgB,EAAE,IAAgB;QAC5D,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;QAC5E,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAA,mBAAY,EAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAA,mBAAY,EAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtH,CAAC;IAED,aAAa,CAAC,IAA2B,EAAE,MAAc,EAAE,IAAY,EAAE,QAAgB;QACvF,KAAK,MAAM,QAAQ,IAAI,IAAI,EAAE,SAAS,IAAI,EAAE,EAAE,CAAC;YAC7C,IAAI,QAAQ,CAAC,IAAI,IAAI,CAAC,IAAA,mBAAY,EAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAS;YACvE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC1D,IAAI,CAAC,IAAI,CAAC,KAAK,MAAM;gBAAE,SAAS;YAChC,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAC9C,IAAI,QAAQ;gBAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAC9C,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;IACtG,CAAC;CACF;;AAED,oGAAoG;AACpG,SAAS,SAAS,CAAC,IAAe,EAAE,GAAc;IAChD,MAAM,KAAK,GAAG,CAAC,CAAY,EAAE,CAAY,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAChH,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,OAAO;QACL,GAAG,GAAG;QACN,GAAG,IAAI;QACP,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE;QACzC,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,WAAW,CAAC;QACrD,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC,SAAS,CAAC;QAC/C,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC;QACnC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC;QACzC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC;QACtC,SAAS,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;KACnG,CAAC;AACJ,CAAC;AAED,oBAA2B,KAAa;IACtC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IACrF,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/B,CAAC;AAED,uFAAuF;AACvF,mBAA0B,OAAe,EAAE,QAAgB;IACzD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACjD,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAW,EAAE;QAC7C,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM;YAAE,OAAO,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC;QAC/C,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACjB,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE;gBAAE,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAAE,OAAO,IAAI,CAAC;YACvE,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACnC,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtC,IAAI,OAAO,EAAE,CAAC;YACZ,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,IAAA,mBAAY,EAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1E,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IACF,OAAO,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3C,CAAC;AAED,sBAA6B,QAAgB,EAAE,QAAgC;IAC7E,OAAO,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,IAAY,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AACnF,CAAC"}

package/dist/src/core/store.d.ts

@@ -0,0 +1,26 @@
+import type { Bundle } from './bundle';
+import { type NormalizedRequest } from './normalize';
+import type { RuleSet } from './rules';
+import type { Entry, StoredResponse } from './types';
+export interface IndexedEntry {
+    entry: Entry;
+    norm: NormalizedRequest;
+    order: number;
+}
+/** In-memory index over a bundle. Keys are recomputed from the current rules at load time. */
+export declare class RecordStore {
+    readonly bundle: Bundle;
+    readonly rules: RuleSet;
+    readonly byKey: Map<string, IndexedEntry[]>;
+    readonly byPathKey: Map<string, IndexedEntry[]>;
+    readonly byId: Map<string, IndexedEntry>;
+    readonly all: IndexedEntry[];
+    private readonly cache;
+    private cacheBytes;
+    constructor(bundle: Bundle, rules: RuleSet);
+    normalizeEntry(entry: Entry): NormalizedRequest;
+    /** Earliest recorded exchange for a URL (exact string match), optionally by method. */
+    findByUrl(url: string, method?: string): IndexedEntry | undefined;
+    response(ix: IndexedEntry): Promise<StoredResponse>;
+    private remember;
+}

package/dist/src/core/store.js

@@ -0,0 +1,78 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecordStore = void 0;
+const normalize_1 = require("./normalize");
+const warc_1 = require("./warc");
+const CACHE_LIMIT_BYTES = 256 * 1024 * 1024;
+/** In-memory index over a bundle. Keys are recomputed from the current rules at load time. */
+class RecordStore {
+    bundle;
+    rules;
+    byKey = new Map();
+    byPathKey = new Map();
+    byId = new Map();
+    all = [];
+    cache = new Map();
+    cacheBytes = 0;
+    constructor(bundle, rules) {
+        this.bundle = bundle;
+        this.rules = rules;
+        const entries = bundle.entries().map((entry, i) => ({ entry, i }));
+        entries.sort((a, b) => (a.entry.ts === b.entry.ts ? a.i - b.i : a.entry.ts < b.entry.ts ? -1 : 1));
+        entries.forEach(({ entry }, order) => {
+            const norm = this.normalizeEntry(entry);
+            const ix = { entry, norm, order };
+            this.all.push(ix);
+            this.byId.set(entry.id, ix);
+            push(this.byKey, norm.key, ix);
+            push(this.byPathKey, norm.pathKey, ix);
+        });
+    }
+    normalizeEntry(entry) {
+        return (0, normalize_1.normalizeRequest)({
+            method: entry.method,
+            url: entry.url,
+            contentType: entry.reqContentType,
+            body: entry.reqBody ? Buffer.from(entry.reqBody, 'base64') : undefined,
+        }, this.rules);
+    }
+    /** Earliest recorded exchange for a URL (exact string match), optionally by method. */
+    findByUrl(url, method = 'GET') {
+        return this.all.find((ix) => ix.entry.url === url && ix.entry.method === method);
+    }
+    async response(ix) {
+        const cached = this.cache.get(ix.entry.id);
+        if (cached)
+            return cached;
+        const own = await (0, warc_1.readResponseAt)(this.bundle.archivePath, { offset: ix.entry.respOffset, length: ix.entry.respLength });
+        let body = own.body;
+        if (ix.entry.revisitOf) {
+            const original = this.byId.get(ix.entry.revisitOf);
+            if (!original)
+                throw new Error(`revisit target missing: ${ix.entry.revisitOf}`);
+            body = (await this.response(original)).body;
+        }
+        const resp = { status: own.status, statusText: own.statusText, headers: own.headers, body };
+        this.remember(ix.entry.id, resp);
+        return resp;
+    }
+    remember(id, resp) {
+        this.cache.set(id, resp);
+        this.cacheBytes += resp.body.length;
+        for (const [k, v] of this.cache) {
+            if (this.cacheBytes <= CACHE_LIMIT_BYTES)
+                break;
+            this.cache.delete(k);
+            this.cacheBytes -= v.body.length;
+        }
+    }
+}
+exports.RecordStore = RecordStore;
+function push(map, key, value) {
+    const list = map.get(key);
+    if (list)
+        list.push(value);
+    else
+        map.set(key, [value]);
+}
+//# sourceMappingURL=store.js.map

package/dist/src/core/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../../../src/core/store.ts"],"names":[],"mappings":";;;AACA,2CAAuE;AAGvE,iCAAwC;AAQxC,MAAM,iBAAiB,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;AAE5C,8FAA8F;AAC9F;IAQuB,MAAM;IAAmB,KAAK;IAP1C,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAC;IAC1C,SAAS,GAAG,IAAI,GAAG,EAA0B,CAAC;IAC9C,IAAI,GAAG,IAAI,GAAG,EAAwB,CAAC;IACvC,GAAG,GAAmB,EAAE,CAAC;IACjB,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAC;IACnD,UAAU,GAAG,CAAC,CAAC;IAEvB,YAAqB,MAAc,EAAW,KAAc;sBAAvC,MAAM;qBAAmB,KAAK;QACjD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnG,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE;YACnC,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YACxC,MAAM,EAAE,GAAiB,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YAChD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC/B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,cAAc,CAAC,KAAY;QACzB,OAAO,IAAA,4BAAgB,EACrB;YACE,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,WAAW,EAAE,KAAK,CAAC,cAAc;YACjC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS;SACvE,EACD,IAAI,CAAC,KAAK,CACX,CAAC;IACJ,CAAC;IAED,uFAAuF;IACvF,SAAS,CAAC,GAAW,EAAE,MAAM,GAAG,KAAK;QACnC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,KAAK,GAAG,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;IACnF,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,EAAgB;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC3C,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAC1B,MAAM,GAAG,GAAG,MAAM,IAAA,qBAAc,EAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;QACxH,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;QACpB,IAAI,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACnD,IAAI,CAAC,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;YAChF,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QACD,MAAM,IAAI,GAAmB,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;QAC5G,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,QAAQ,CAAC,EAAU,EAAE,IAAoB;QAC/C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QACzB,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QACpC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,UAAU,IAAI,iBAAiB;gBAAE,MAAM;YAChD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACrB,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QACnC,CAAC;IACH,CAAC;CACF;;AAED,SAAS,IAAI,CAAO,GAAgB,EAAE,GAAM,EAAE,KAAQ;IACpD,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,IAAI;QAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;QACtB,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;AAC7B,CAAC"}

package/dist/src/core/types.d.ts

@@ -0,0 +1,125 @@
+export type HeaderList = [string, string][];
+/** One HTTP request/response pair as captured from a real site. */
+export interface Exchange {
+    id: string;
+    ts: string;
+    method: string;
+    url: string;
+    httpVersion: string;
+    reqHeaders: HeaderList;
+    reqBody?: Buffer;
+    status: number;
+    statusText: string;
+    respHeaders: HeaderList;
+    respBody: Buffer;
+    /** Content-Encoding the site used on the wire; the stored body is already decoded. */
+    originalContentEncoding?: string;
+    resourceType?: string;
+    /** Read from a WARC: stored header lines of the request and response that are not headers (see parseHeaderLines). */
+    malformedHeaderLines?: number;
+}
+/** Index row for one exchange inside a bundle (indexes/entries.jsonl). */
+export interface Entry {
+    id: string;
+    ts: string;
+    method: string;
+    url: string;
+    status: number;
+    mime: string;
+    resourceType?: string;
+    reqContentType?: string;
+    /** Request body, base64, kept inline so keys can be recomputed without reading the WARC. */
+    reqBody?: string;
+    digest: string;
+    size: number;
+    reqOffset: number;
+    reqLength: number;
+    respOffset: number;
+    respLength: number;
+    /** Set when this response was stored as a revisit of an identical earlier payload. */
+    revisitOf?: string;
+}
+export type Kind = 'browse' | 'search' | 'promo' | 'stateful' | 'block' | 'static';
+export interface EndpointRule {
+    name: string;
+    /** "METHOD /path/pattern" or "/path/pattern"; `*` one segment, `**` any segments, `{x}` captures one segment. */
+    match: string;
+    /** Host glob this endpoint is limited to; defaults to any host of the site. */
+    host?: string;
+    /** Template for the normalized path, e.g. "/dp/{asin}". */
+    key?: string;
+    kind?: Kind;
+    keep_params?: string[];
+    drop_params?: string[];
+    body_keep?: string[];
+    body_drop?: string[];
+    /** Query or body field holding free text (search) or a code (promo). */
+    text_param?: string;
+    /** Minimum token similarity for search matching (0-1). */
+    similarity?: number;
+    /** Allow same-path matching when the exact key is missing. GET defaults to true, bodies to false. */
+    fuzzy?: boolean;
+    /** Minimum share of query parameters that must agree for a fuzzy match. */
+    fuzzy_min?: number;
+    /** For promo endpoints: a code recorded as invalid, whose response answers unknown codes. */
+    invalid_code?: string;
+}
+export interface SiteRules {
+    site: string;
+    hosts: string[];
+    drop_params?: string[];
+    body_drop?: string[];
+    block?: string[];
+    /** Never stored: bot checks and other pages that must not be replayed ("host/path" globs). */
+    exclude?: string[];
+    /**
+     * Pages whose content follows what was done before them, such as the cart ("host/path" globs):
+     * replay answers them with the last recording instead of the first.
+     */
+    latest?: string[];
+    endpoints?: EndpointRule[];
+    /** URL of this site's own 404 page recorded during capture. */
+    not_found?: string;
+    /** Body returned for missed API requests of this site. */
+    xhr_not_found?: {
+        content_type: string;
+        body: string;
+    };
+}
+/** manifest.json: only what replay and import read. */
+export interface Manifest {
+    /** Benchmark task this recording belongs to (tasks/<task_id>/). */
+    task_id: string;
+    /** Page clocks start here during replay. */
+    captured_from: string;
+    captured_to: string;
+    /** Each site's own 404 page, answered for documents replay cannot find. */
+    not_found: Record<string, string>;
+    /** Replay browsers use the recording's language and time zone. */
+    locale?: string;
+    timezone?: string;
+    /** Layout version of a recording published in a task (RECORDING_FORMAT); absent elsewhere. */
+    format?: number;
+}
+export interface StoredResponse {
+    status: number;
+    statusText: string;
+    headers: HeaderList;
+    body: Buffer;
+}
+export type Source = 'exact' | 'fuzzy' | 'search' | 'promo' | 'blocked' | 'preflight' | 'control' | 'miss';
+export type ResourceClass = 'document' | 'xhr' | 'static' | 'tracker';
+export interface ReplayRequest {
+    method: string;
+    url: string;
+    headers: Record<string, string | string[] | undefined>;
+    body?: Buffer;
+}
+export interface ReplayResponse extends StoredResponse {
+    source: Source;
+    key: string;
+    kind: Kind;
+    resourceClass: ResourceClass;
+    entryId?: string;
+    detail?: string;
+}

package/dist/src/core/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/src/core/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/core/types.ts"],"names":[],"mappings":""}