personal-agent-benchmark 0.0.0-stage → 0.2.0

package/dist/src/core/util.d.ts

@@ -0,0 +1,21 @@
+import type { HeaderList } from './types';
+export declare function sha256(data: Buffer | string): string;
+/** JSON.stringify with sorted object keys, so equal objects give equal strings. */
+export declare function stableStringify(value: unknown): string;
+/** Glob with `*` (anything) and `?` (one char); case-insensitive. */
+export declare function globToRegExp(glob: string): RegExp;
+export declare function matchesAny(value: string, globs: string[] | undefined): boolean;
+export declare function header(headers: HeaderList, name: string): string | undefined;
+export declare function headerFromRecord(headers: Record<string, string | string[] | undefined>, name: string): string | undefined;
+export declare function withoutHeaders(headers: HeaderList, names: string[]): HeaderList;
+export declare function setHeader(headers: HeaderList, name: string, value: string): HeaderList;
+export declare function mimeOf(headers: HeaderList): string;
+export declare function isTextMime(mime: string): boolean;
+export declare function readJsonl<T>(file: string): T[];
+export declare function writeJsonl(file: string, rows: unknown[]): void;
+export declare function writeJson(file: string, value: unknown): void;
+export declare function readJson<T>(file: string): T;
+/** 14-digit WARC/CDX timestamp from an ISO date. */
+export declare function cdxTimestamp(iso: string): string;
+export declare function tokens(text: string): string[];
+export declare function jaccard(a: string[], b: string[]): number;

package/dist/src/core/util.js

@@ -0,0 +1,150 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sha256 = sha256;
+exports.stableStringify = stableStringify;
+exports.globToRegExp = globToRegExp;
+exports.matchesAny = matchesAny;
+exports.header = header;
+exports.headerFromRecord = headerFromRecord;
+exports.withoutHeaders = withoutHeaders;
+exports.setHeader = setHeader;
+exports.mimeOf = mimeOf;
+exports.isTextMime = isTextMime;
+exports.readJsonl = readJsonl;
+exports.writeJsonl = writeJsonl;
+exports.writeJson = writeJson;
+exports.readJson = readJson;
+exports.cdxTimestamp = cdxTimestamp;
+exports.tokens = tokens;
+exports.jaccard = jaccard;
+const node_crypto_1 = require("node:crypto");
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+function sha256(data) {
+    return (0, node_crypto_1.createHash)('sha256').update(data).digest('hex');
+}
+/** JSON.stringify with sorted object keys, so equal objects give equal strings. */
+function stableStringify(value) {
+    if (value === null || typeof value !== 'object')
+        return JSON.stringify(value);
+    if (Array.isArray(value))
+        return '[' + value.map(stableStringify).join(',') + ']';
+    const obj = value;
+    return '{' + Object.keys(obj).sort().map((k) => JSON.stringify(k) + ':' + stableStringify(obj[k])).join(',') + '}';
+}
+/** Glob with `*` (anything) and `?` (one char); case-insensitive. */
+function globToRegExp(glob) {
+    const re = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\?/g, '.');
+    return new RegExp('^' + re + '$', 'i');
+}
+function matchesAny(value, globs) {
+    if (!globs || globs.length === 0)
+        return false;
+    return globs.some((g) => globToRegExp(g).test(value));
+}
+function header(headers, name) {
+    const lower = name.toLowerCase();
+    for (const [k, v] of headers)
+        if (k.toLowerCase() === lower)
+            return v;
+    return undefined;
+}
+function headerFromRecord(headers, name) {
+    const lower = name.toLowerCase();
+    for (const k of Object.keys(headers)) {
+        if (k.toLowerCase() !== lower)
+            continue;
+        const v = headers[k];
+        return Array.isArray(v) ? v.join(', ') : v;
+    }
+    return undefined;
+}
+function withoutHeaders(headers, names) {
+    const drop = new Set(names.map((n) => n.toLowerCase()));
+    return headers.filter(([k]) => !drop.has(k.toLowerCase()));
+}
+function setHeader(headers, name, value) {
+    return [...withoutHeaders(headers, [name]), [name, value]];
+}
+function mimeOf(headers) {
+    return (header(headers, 'content-type') || '').split(';')[0].trim().toLowerCase();
+}
+function isTextMime(mime) {
+    return (mime.startsWith('text/') ||
+        /(json|javascript|xml|x-www-form-urlencoded|svg)/.test(mime));
+}
+function readJsonl(file) {
+    if (!fs.existsSync(file))
+        return [];
+    return fs
+        .readFileSync(file, 'utf8')
+        .split('\n')
+        .filter((l) => l.trim())
+        .map((l) => JSON.parse(l));
+}
+function writeJsonl(file, rows) {
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    fs.writeFileSync(file, rows.map((r) => JSON.stringify(r)).join('\n') + (rows.length ? '\n' : ''));
+}
+function writeJson(file, value) {
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n');
+}
+function readJson(file) {
+    return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+/** 14-digit WARC/CDX timestamp from an ISO date. */
+function cdxTimestamp(iso) {
+    return new Date(iso).toISOString().replace(/[-:TZ.]/g, '').slice(0, 14);
+}
+function tokens(text) {
+    return text
+        .toLowerCase()
+        .split(/[^\p{L}\p{N}]+/u)
+        .filter(Boolean);
+}
+function jaccard(a, b) {
+    const sa = new Set(a);
+    const sb = new Set(b);
+    if (sa.size === 0 && sb.size === 0)
+        return 1;
+    let inter = 0;
+    for (const t of sa)
+        if (sb.has(t))
+            inter++;
+    return inter / (sa.size + sb.size - inter);
+}
+//# sourceMappingURL=util.js.map

package/dist/src/core/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/core/util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAAyC;AACzC,MAAY,EAAE,oCAAgB;AAC9B,MAAY,IAAI,sCAAkB;AAGlC,gBAAuB,IAAqB;IAC1C,OAAO,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACzD,CAAC;AAED,mFAAmF;AACnF,yBAAgC,KAAc;IAC5C,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9E,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IAClF,MAAM,GAAG,GAAG,KAAgC,CAAC;IAC7C,OAAO,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;AACrH,CAAC;AAED,qEAAqE;AACrE,sBAA6B,IAAY;IACvC,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC9F,OAAO,IAAI,MAAM,CAAC,GAAG,GAAG,EAAE,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;AACzC,CAAC;AAED,oBAA2B,KAAa,EAAE,KAA2B;IACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/C,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,gBAAuB,OAAmB,EAAE,IAAY;IACtD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO;QAAE,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,KAAK;YAAE,OAAO,CAAC,CAAC;IACtE,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,0BAAiC,OAAsD,EAAE,IAAY;IACnG,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,KAAK;YAAE,SAAS;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,wBAA+B,OAAmB,EAAE,KAAe;IACjE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACxD,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED,mBAA0B,OAAmB,EAAE,IAAY,EAAE,KAAa;IACxE,OAAO,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED,gBAAuB,OAAmB;IACxC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACpF,CAAC;AAED,oBAA2B,IAAY;IACrC,OAAO,CACL,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QACxB,iDAAiD,CAAC,IAAI,CAAC,IAAI,CAAC,CAC7D,CAAC;AACJ,CAAC;AAED,mBAA6B,IAAY;IACvC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACpC,OAAO,EAAE;SACN,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC;SAC1B,KAAK,CAAC,IAAI,CAAC;SACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACvB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAM,CAAC,CAAC;AACpC,CAAC;AAED,oBAA2B,IAAY,EAAE,IAAe;IACtD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACpG,CAAC;AAED,mBAA0B,IAAY,EAAE,KAAc;IACpD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;AAChE,CAAC;AAED,kBAA4B,IAAY;IACtC,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAM,CAAC;AACxD,CAAC;AAED,oDAAoD;AACpD,sBAA6B,GAAW;IACtC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC1E,CAAC;AAED,gBAAuB,IAAY;IACjC,OAAO,IAAI;SACR,WAAW,EAAE;SACb,KAAK,CAAC,iBAAiB,CAAC;SACxB,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC;AAED,iBAAwB,CAAW,EAAE,CAAW;IAC9C,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;IACtB,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;IACtB,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,EAAE;QAAE,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,KAAK,EAAE,CAAC;IAC3C,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC;AAC7C,CAAC"}

package/dist/src/core/warc.d.ts

@@ -0,0 +1,55 @@
+import type { Exchange, HeaderList, StoredResponse } from './types';
+export interface Span {
+    offset: number;
+    length: number;
+}
+/**
+ * Appends WARC records to one file and reports where each landed. Records are written
+ * uncompressed by default: bundles are compressed as a whole for transport, which shrinks them
+ * far more than per-record gzip (cross-record redundancy); pass gzip for a standalone .warc.gz.
+ */
+export declare class WarcWriter {
+    readonly file: string;
+    private readonly opts;
+    private readonly fd;
+    private offset;
+    constructor(file: string, opts?: {
+        gzip?: boolean;
+    });
+    writeInfo(info: Record<string, string>): Promise<Span>;
+    /** Writes a response (or a revisit pointing at an identical earlier payload) and its request. */
+    writeExchange(ex: Exchange, revisit?: {
+        url: string;
+        date: string;
+        digest: string;
+    }): Promise<{
+        req: Span;
+        resp: Span;
+    }>;
+    close(): void;
+    private write;
+}
+/**
+ * Header lines as stored, as a header list. Recorders before record-replay 0.5.5 kept a header the
+ * site sent several times (Set-Cookie above all) as one value joined by newlines, as DevTools
+ * reports it, and wrote it that way: each further value is a line of its own after the header's
+ * line, ended by a bare newline, or by CRLF once the recording was rewritten. A line that is not a
+ * header is taken as a further value of the header before it; `malformed` counts those lines (lines
+ * before any header are left out).
+ */
+export declare function parseHeaderLines(lines: string[]): {
+    headers: HeaderList;
+    malformed: number;
+};
+/** Reads one record (plain or gzip member) at a known offset and returns its HTTP status, headers and payload. */
+export declare function readResponseAt(file: string, span: Span): Promise<StoredResponse & {
+    type: string;
+    malformedHeaderLines: number;
+}>;
+export declare function readRequestAt(file: string, span: Span): Promise<{
+    headers: HeaderList;
+    malformedHeaderLines: number;
+    body: Buffer;
+}>;
+/** Iterates request/response pairs of any WARC (e.g. from ArchiveWeb.page) as exchanges. */
+export declare function readExchanges(file: string): AsyncGenerator<Exchange>;

package/dist/src/core/warc.js

@@ -0,0 +1,258 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WarcWriter = void 0;
+exports.parseHeaderLines = parseHeaderLines;
+exports.readResponseAt = readResponseAt;
+exports.readRequestAt = readRequestAt;
+exports.readExchanges = readExchanges;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const node_http_1 = require("node:http");
+const warcio_1 = require("warcio");
+const util_1 = require("./util");
+const REVISIT_PROFILE = 'http://netpreserve.org/warc/1.1/revisit/identical-payload-digest';
+/**
+ * Appends WARC records to one file and reports where each landed. Records are written
+ * uncompressed by default: bundles are compressed as a whole for transport, which shrinks them
+ * far more than per-record gzip (cross-record redundancy); pass gzip for a standalone .warc.gz.
+ */
+class WarcWriter {
+    file;
+    opts;
+    fd;
+    offset = 0;
+    constructor(file, opts = {}) {
+        this.file = file;
+        this.opts = opts;
+        fs.mkdirSync(path.dirname(file), { recursive: true });
+        this.fd = fs.openSync(file, 'w');
+    }
+    async writeInfo(info) {
+        const record = warcio_1.WARCRecord.createWARCInfo({ filename: path.basename(this.file), warcVersion: 'WARC/1.1' }, info);
+        return this.write(record);
+    }
+    /** Writes a response (or a revisit pointing at an identical earlier payload) and its request. */
+    async writeExchange(ex, revisit) {
+        const statusline = `HTTP/1.1 ${ex.status} ${ex.statusText || node_http_1.STATUS_CODES[ex.status] || ''}`.trim();
+        const warcHeaders = { 'RR-Exchange-Id': ex.id };
+        if (ex.originalContentEncoding)
+            warcHeaders['RR-Original-Content-Encoding'] = ex.originalContentEncoding;
+        if (ex.resourceType)
+            warcHeaders['RR-Resource-Type'] = ex.resourceType;
+        let respRecord;
+        if (revisit) {
+            respRecord = warcio_1.WARCRecord.create({
+                url: ex.url,
+                date: ex.ts,
+                type: 'revisit',
+                warcVersion: 'WARC/1.1',
+                refersToUrl: revisit.url,
+                refersToDate: revisit.date,
+                warcHeaders: { ...warcHeaders, 'WARC-Profile': REVISIT_PROFILE, 'RR-Payload-Sha256': revisit.digest },
+                httpHeaders: ex.respHeaders,
+                statusline,
+                keepHeadersCase: true,
+            }, []);
+        }
+        else {
+            respRecord = warcio_1.WARCRecord.create({
+                url: ex.url,
+                date: ex.ts,
+                type: 'response',
+                warcVersion: 'WARC/1.1',
+                warcHeaders,
+                httpHeaders: ex.respHeaders,
+                statusline,
+                keepHeadersCase: true,
+            }, [ex.respBody]);
+        }
+        const respId = respRecord.warcHeader('WARC-Record-ID') ?? '';
+        const resp = await this.write(respRecord);
+        const u = new URL(ex.url);
+        const reqHeaders = (0, util_1.header)(ex.reqHeaders, 'host') ? ex.reqHeaders : [['Host', u.host], ...ex.reqHeaders];
+        const reqRecord = warcio_1.WARCRecord.create({
+            url: ex.url,
+            date: ex.ts,
+            type: 'request',
+            warcVersion: 'WARC/1.1',
+            warcHeaders: { 'WARC-Concurrent-To': respId, 'RR-Exchange-Id': ex.id },
+            httpHeaders: reqHeaders,
+            statusline: `${ex.method} ${u.pathname}${u.search} HTTP/1.1`,
+            keepHeadersCase: true,
+        }, ex.reqBody && ex.reqBody.length ? [ex.reqBody] : []);
+        const req = await this.write(reqRecord);
+        return { req, resp };
+    }
+    close() {
+        fs.closeSync(this.fd);
+    }
+    async write(record) {
+        const bytes = await warcio_1.WARCSerializer.serialize(record, { gzip: this.opts.gzip ?? false });
+        fs.writeSync(this.fd, bytes);
+        const span = { offset: this.offset, length: bytes.length };
+        this.offset += bytes.length;
+        return span;
+    }
+}
+exports.WarcWriter = WarcWriter;
+const HEADER_NAME = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/;
+const CRLFCRLF = Buffer.from('\r\n\r\n');
+/**
+ * Header lines as stored, as a header list. Recorders before record-replay 0.5.5 kept a header the
+ * site sent several times (Set-Cookie above all) as one value joined by newlines, as DevTools
+ * reports it, and wrote it that way: each further value is a line of its own after the header's
+ * line, ended by a bare newline, or by CRLF once the recording was rewritten. A line that is not a
+ * header is taken as a further value of the header before it; `malformed` counts those lines (lines
+ * before any header are left out).
+ */
+function parseHeaderLines(lines) {
+    const headers = [];
+    let malformed = 0;
+    let last;
+    for (const line of lines) {
+        // values joined to a line by bare newlines are never headers of their own
+        for (const [i, part] of line.split(/\r?\n|\r/).entries()) {
+            const colon = part.indexOf(':');
+            const name = colon > 0 ? part.slice(0, colon) : '';
+            if (i === 0 && HEADER_NAME.test(name)) {
+                headers.push([name, part.slice(colon + 1).trim()]);
+                last = name;
+                continue;
+            }
+            malformed++;
+            if (last && part.trim())
+                headers.push([last, part.trim()]);
+        }
+    }
+    return { headers, malformed };
+}
+/**
+ * The HTTP message stored in a record, read by its bytes: warcio's own parsing drops header lines
+ * without a colon, so a malformed header could be neither repaired nor seen.
+ */
+function httpMessage(content) {
+    const end = content.indexOf(CRLFCRLF);
+    const [startLine = '', ...lines] = (end < 0 ? content : content.subarray(0, end)).toString('utf8').split('\r\n');
+    const { headers, malformed } = parseHeaderLines(lines.filter(Boolean));
+    return { startLine, headers, malformedHeaderLines: malformed, body: end < 0 ? Buffer.alloc(0) : content.subarray(end + 4) };
+}
+/** Status of a stored response; a request that got no response is stored with status -1. */
+function statusOf(startLine) {
+    const m = /^HTTP\/\S+ (-?\d+) ?(.*)$/.exec(startLine);
+    return m ? { status: Number(m[1]), statusText: m[2] } : { status: 200, statusText: '' };
+}
+async function readRecordAt(file, span) {
+    const record = await warcio_1.WARCParser.parse([readSpan(file, span)], { keepHeadersCase: true, parseHttp: false });
+    if (!record)
+        throw new Error(`no WARC record at ${file}:${span.offset}`);
+    return { type: record.warcType ?? '', http: httpMessage(Buffer.from(await record.readFully(false))) };
+}
+/** Reads one record (plain or gzip member) at a known offset and returns its HTTP status, headers and payload. */
+async function readResponseAt(file, span) {
+    const { type, http } = await readRecordAt(file, span);
+    return { type, ...statusOf(http.startLine), headers: http.headers, malformedHeaderLines: http.malformedHeaderLines, body: http.body };
+}
+async function readRequestAt(file, span) {
+    const { http } = await readRecordAt(file, span);
+    return { headers: http.headers, malformedHeaderLines: http.malformedHeaderLines, body: http.body };
+}
+function readSpan(file, span) {
+    const fd = fs.openSync(file, 'r');
+    try {
+        const buf = Buffer.alloc(span.length);
+        fs.readSync(fd, buf, 0, span.length, span.offset);
+        return buf;
+    }
+    finally {
+        fs.closeSync(fd);
+    }
+}
+/** Iterates request/response pairs of any WARC (e.g. from ArchiveWeb.page) as exchanges. */
+async function* readExchanges(file) {
+    const stream = fs.createReadStream(file);
+    let n = 0;
+    const parser = new warcio_1.WARCParser(stream, { keepHeadersCase: true, parseHttp: false });
+    const responses = [];
+    const byRespId = new Map();
+    const requests = [];
+    for await (const record of parser) {
+        const type = record.warcType;
+        if (type === 'response' || type === 'revisit') {
+            const http = httpMessage(Buffer.from(await record.readFully(false)));
+            const ex = {
+                id: `w${n++}`,
+                ts: record.warcDate ?? new Date().toISOString(),
+                method: 'GET',
+                url: record.warcTargetURI ?? '',
+                httpVersion: 'HTTP/1.1',
+                reqHeaders: [],
+                ...statusOf(http.startLine),
+                respHeaders: http.headers,
+                respBody: type === 'response' ? http.body : Buffer.alloc(0),
+                originalContentEncoding: (0, util_1.header)(http.headers, 'content-encoding'),
+                malformedHeaderLines: http.malformedHeaderLines,
+            };
+            responses.push(ex);
+            byRespId.set(record.warcHeader('WARC-Record-ID') ?? '', ex);
+        }
+        else if (type === 'request') {
+            const http = httpMessage(Buffer.from(await record.readFully(false)));
+            requests.push({
+                concurrentTo: record.warcHeader('WARC-Concurrent-To') ?? undefined,
+                url: record.warcTargetURI ?? '',
+                method: http.startLine.split(' ')[0] || 'GET',
+                headers: http.headers,
+                malformedHeaderLines: http.malformedHeaderLines,
+                body: http.body,
+            });
+        }
+        else {
+            await record.skipFully();
+        }
+    }
+    for (const r of requests) {
+        const ex = (r.concurrentTo && byRespId.get(r.concurrentTo)) || responses.find((e) => e.url === r.url && e.reqHeaders.length === 0);
+        if (!ex)
+            continue;
+        ex.method = r.method.toUpperCase();
+        ex.reqHeaders = r.headers;
+        ex.malformedHeaderLines = (ex.malformedHeaderLines ?? 0) + r.malformedHeaderLines;
+        ex.reqBody = r.body.length ? r.body : undefined;
+    }
+    for (const ex of responses)
+        yield ex;
+}
+//# sourceMappingURL=warc.js.map

package/dist/src/core/warc.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"warc.js","sourceRoot":"","sources":["../../../src/core/warc.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,MAAY,EAAE,oCAAgB;AAC9B,MAAY,IAAI,sCAAkB;AAClC,yCAAyC;AACzC,mCAAgE;AAEhE,iCAAgC;AAOhC,MAAM,eAAe,GAAG,kEAAkE,CAAC;AAE3F;;;;GAIG;AACH;IAIuB,IAAI;IAA2B,IAAI;IAHvC,EAAE,CAAS;IACpB,MAAM,GAAG,CAAC,CAAC;IAEnB,YAAqB,IAAY,EAAmB,IAAI,GAAuB,EAAE;oBAA5D,IAAI;oBAA2B,IAAI;QACtD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAA4B;QAC1C,MAAM,MAAM,GAAG,mBAAU,CAAC,cAAc,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,EAAE,IAAI,CAAC,CAAC;QAChH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED,iGAAiG;IACjG,KAAK,CAAC,aAAa,CAAC,EAAY,EAAE,OAAuD;QACvF,MAAM,UAAU,GAAG,YAAY,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,UAAU,IAAI,wBAAY,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;QACpG,MAAM,WAAW,GAA2B,EAAE,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QACxE,IAAI,EAAE,CAAC,uBAAuB;YAAE,WAAW,CAAC,8BAA8B,CAAC,GAAG,EAAE,CAAC,uBAAuB,CAAC;QACzG,IAAI,EAAE,CAAC,YAAY;YAAE,WAAW,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC;QAEvE,IAAI,UAAsB,CAAC;QAC3B,IAAI,OAAO,EAAE,CAAC;YACZ,UAAU,GAAG,mBAAU,CAAC,MAAM,CAC5B;gBACE,GAAG,EAAE,EAAE,CAAC,GAAG;gBACX,IAAI,EAAE,EAAE,CAAC,EAAE;gBACX,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,UAAU;gBACvB,WAAW,EAAE,OAAO,CAAC,GAAG;gBACxB,YAAY,EAAE,OAAO,CAAC,IAAI;gBAC1B,WAAW,EAAE,EAAE,GAAG,WAAW,EAAE,cAAc,EAAE,eAAe,EAAE,mBAAmB,EAAE,OAAO,CAAC,MAAM,EAAE;gBACrG,WAAW,EAAE,EAAE,CAAC,WAAW;gBAC3B,UAAU;gBACV,eAAe,EAAE,IAAI;aACtB,EACD,EAAE,CACH,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,UAAU,GAAG,mBAAU,CAAC,MAAM,CAC5B;gBACE,GAAG,EAAE,EAAE,CAAC,GAAG;gBACX,IAAI,EAAE,EAAE,CAAC,EAAE;gBACX,IAAI,EAAE,UAAU;gBAChB,WAAW,EAAE,UAAU;gBACvB,WAAW;gBACX,WAAW,EAAE,EAAE,CAAC,WAAW;gBAC3B,UAAU;gBACV,eAAe,EAAE,IAAI;aACtB,EACD,CAAC,EAAE,CAAC,QAAQ,CAAC,CACd,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,UAAU,CAAC,UAAU,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAC7D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAE1C,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QAC1B,MAAM,UAAU,GAAe,IAAA,aAAM,EAAC,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,CAAC;QACpH,MAAM,SAAS,GAAG,mBAAU,CAAC,MAAM,CACjC;YACE,GAAG,EAAE,EAAE,CAAC,GAAG;YACX,IAAI,EAAE,EAAE,CAAC,EAAE;YACX,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,UAAU;YACvB,WAAW,EAAE,EAAE,oBAAoB,EAAE,MAAM,EAAE,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE;YACtE,WAAW,EAAE,UAAU;YACvB,UAAU,EAAE,GAAG,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,MAAM,WAAW;YAC5D,eAAe,EAAE,IAAI;SACtB,EACD,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CACpD,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACxC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;IACvB,CAAC;IAED,KAAK;QACH,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,MAAkB;QACpC,MAAM,KAAK,GAAG,MAAM,uBAAc,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC;QACxF,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7B,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;QAC3D,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;CACF;;AAED,MAAM,WAAW,GAAG,+BAA+B,CAAC;AACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AAEzC;;;;;;;GAOG;AACH,0BAAiC,KAAe;IAC9C,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,IAAwB,CAAC;IAC7B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,0EAA0E;QAC1E,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YACzD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChC,MAAM,IAAI,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBACnD,IAAI,GAAG,IAAI,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,SAAS,EAAE,CAAC;YACZ,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE;gBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAWD;;;GAGG;AACH,SAAS,WAAW,CAAC,OAAe;IAClC,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,CAAC,SAAS,GAAG,EAAE,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACjH,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;IACvE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,oBAAoB,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC;AAC9H,CAAC;AAED,4FAA4F;AAC5F,SAAS,QAAQ,CAAC,SAAiB;IACjC,MAAM,CAAC,GAAG,2BAA2B,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtD,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;AAC1F,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,IAAU;IAClD,MAAM,MAAM,GAAG,MAAM,mBAAU,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3G,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IACzE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,QAAQ,IAAI,EAAE,EAAE,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AACxG,CAAC;AAED,kHAAkH;AAC3G,KAAK,yBAAyB,IAAY,EAAE,IAAU;IAC3D,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACtD,OAAO,EAAE,IAAI,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;AACxI,CAAC;AAEM,KAAK,wBAAwB,IAAY,EAAE,IAAU;IAC1D,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAChD,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;AACrG,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,IAAU;IACxC,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtC,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClD,OAAO,GAAG,CAAC;IACb,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACnB,CAAC;AACH,CAAC;AAED,4FAA4F;AACrF,KAAK,SAAS,CAAC,eAAe,IAAY;IAC/C,MAAM,MAAM,GAAG,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,MAAM,GAAG,IAAI,mBAAU,CAAC,MAAM,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;IACnF,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC7C,MAAM,QAAQ,GAA8H,EAAE,CAAC;IAC/I,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC7B,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACrE,MAAM,EAAE,GAAa;gBACnB,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE;gBACb,EAAE,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBAC/C,MAAM,EAAE,KAAK;gBACb,GAAG,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;gBAC/B,WAAW,EAAE,UAAU;gBACvB,UAAU,EAAE,EAAE;gBACd,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;gBAC3B,WAAW,EAAE,IAAI,CAAC,OAAO;gBACzB,QAAQ,EAAE,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC3D,uBAAuB,EAAE,IAAA,aAAM,EAAC,IAAI,CAAC,OAAO,EAAE,kBAAkB,CAAC;gBACjE,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;aAChD,CAAC;YACF,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnB,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9D,CAAC;aAAM,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACrE,QAAQ,CAAC,IAAI,CAAC;gBACZ,YAAY,EAAE,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,IAAI,SAAS;gBAClE,GAAG,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;gBAC/B,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK;gBAC7C,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;gBAC/C,IAAI,EAAE,IAAI,CAAC,IAAI;aAChB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,YAAY,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;QACnI,IAAI,CAAC,EAAE;YAAE,SAAS;QAClB,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACnC,EAAE,CAAC,UAAU,GAAG,CAAC,CAAC,OAAO,CAAC;QAC1B,EAAE,CAAC,oBAAoB,GAAG,CAAC,EAAE,CAAC,oBAAoB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAC;QAClF,EAAE,CAAC,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IAClD,CAAC;IACD,KAAK,MAAM,EAAE,IAAI,SAAS;QAAE,MAAM,EAAE,CAAC;AACvC,CAAC"}

package/dist/src/core/writer.d.ts

@@ -0,0 +1,26 @@
+import type { Bundle, PageRecord } from './bundle';
+import type { RuleSet } from './rules';
+import type { Entry, Exchange, Manifest } from './types';
+/** The manifest fields a recorder knows before writing; the rest come from the exchanges. */
+export type BundleMeta = Pick<Manifest, 'task_id' | 'locale' | 'timezone'>;
+export interface WriteBundleOptions {
+    meta: BundleMeta;
+    /** Decides site names and which page is each site's 404 page. */
+    rules: RuleSet;
+    /** Top-level pages visited while recording; defaults to every HTML document. */
+    pages?: PageRecord[];
+    /** Payloads at least this large are stored once and referenced by revisit records. */
+    dedupeMinBytes?: number;
+}
+export interface WriteBundleResult {
+    entries: Entry[];
+    manifest: Manifest;
+    revisits: number;
+    bytes: number;
+}
+/**
+ * Writes the replayable part of a bundle from exchanges that are already safe to store:
+ * archive/data.warc, indexes/entries.jsonl, pages/pages.jsonl and manifest.json. Exchanges are
+ * stored in the given order and get ids e000001, e000002, ...
+ */
+export declare function writeBundle(bundle: Bundle, exchanges: Exchange[], o: WriteBundleOptions): Promise<WriteBundleResult>;

package/dist/src/core/writer.js

@@ -0,0 +1,119 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeBundle = writeBundle;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const util_1 = require("./util");
+const warc_1 = require("./warc");
+/** Files of earlier bundle versions that replay never read; removed when a bundle is rewritten. */
+const LEGACY_FILES = ['datapackage.json', path.join('indexes', 'index.cdxj')];
+/**
+ * Writes the replayable part of a bundle from exchanges that are already safe to store:
+ * archive/data.warc, indexes/entries.jsonl, pages/pages.jsonl and manifest.json. Exchanges are
+ * stored in the given order and get ids e000001, e000002, ...
+ */
+async function writeBundle(bundle, exchanges, o) {
+    if (!exchanges.length)
+        throw new Error('no exchanges to write');
+    exchanges.forEach((ex, i) => (ex.id = 'e' + String(i + 1).padStart(6, '0')));
+    fs.rmSync(path.join(bundle.dir, 'archive'), { recursive: true, force: true });
+    for (const f of LEGACY_FILES)
+        fs.rmSync(path.join(bundle.dir, f), { force: true });
+    const writer = new warc_1.WarcWriter(path.join(bundle.dir, 'archive', 'data.warc'));
+    const firstByDigest = new Map();
+    const minDedupe = o.dedupeMinBytes ?? 1024;
+    const entries = [];
+    let revisits = 0;
+    let bytes = 0;
+    for (const ex of exchanges) {
+        const digest = (0, util_1.sha256)(ex.respBody);
+        const first = ex.respBody.length >= minDedupe ? firstByDigest.get(digest) : undefined;
+        const spans = await writer.writeExchange(ex, first ? { url: first.url, date: first.ts, digest } : undefined);
+        if (first)
+            revisits++;
+        else {
+            bytes += ex.respBody.length;
+            if (ex.respBody.length >= minDedupe)
+                firstByDigest.set(digest, ex);
+        }
+        entries.push({
+            id: ex.id,
+            ts: ex.ts,
+            method: ex.method,
+            url: ex.url,
+            status: ex.status,
+            mime: (0, util_1.mimeOf)(ex.respHeaders),
+            resourceType: ex.resourceType,
+            reqContentType: (0, util_1.header)(ex.reqHeaders, 'content-type'),
+            reqBody: ex.reqBody && ex.reqBody.length ? ex.reqBody.toString('base64') : undefined,
+            digest,
+            size: ex.respBody.length,
+            reqOffset: spans.req.offset,
+            reqLength: spans.req.length,
+            respOffset: spans.resp.offset,
+            respLength: spans.resp.length,
+            revisitOf: first?.id,
+        });
+    }
+    writer.close();
+    (0, util_1.writeJsonl)(bundle.entriesPath, entries);
+    const pages = o.pages?.length
+        ? o.pages
+        : entries
+            .filter((e) => e.method === 'GET' && e.status === 200 && e.mime === 'text/html' && (e.resourceType ?? 'document') === 'document')
+            .map((e) => ({ id: e.id, url: e.url, ts: e.ts }));
+    const getUrls = new Set(entries.filter((e) => e.method === 'GET').map((e) => e.url));
+    const otherUrls = new Set(entries.filter((e) => e.method !== 'GET').map((e) => e.url));
+    for (const p of pages)
+        p.fetched = getUrls.has(p.url) ? 'GET' : otherUrls.has(p.url) ? 'POST' : 'none';
+    (0, util_1.writeJsonl)(bundle.pagesPath, pages);
+    const notFound = {};
+    for (const p of pages.filter((pg) => pg.note === 'not-found-probe')) {
+        const host = new URL(p.url).hostname;
+        notFound[o.rules.siteFor(host)?.site ?? host] ??= p.url;
+    }
+    const manifest = {
+        task_id: o.meta.task_id,
+        captured_from: entries[0].ts,
+        captured_to: entries[entries.length - 1].ts,
+        not_found: notFound,
+        locale: o.meta.locale,
+        timezone: o.meta.timezone,
+    };
+    (0, util_1.writeJson)(bundle.manifestPath, manifest);
+    return { entries, manifest, revisits, bytes };
+}
+//# sourceMappingURL=writer.js.map