polite-crawler 0.1.0 → 0.1.2

package/build/warc-VIPscWxY.mjs

@@ -0,0 +1,728 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { n as FileBlobStore, t as CaptureLog } from "./log-Dx-wXjJl.mjs";
+import { i as VERSION, n as BOT_PAGE } from "./duration-iIuMMofi.mjs";
+import { createHash, randomUUID } from "node:crypto";
+import { open, rename, rm, stat } from "node:fs/promises";
+import { basename, join, resolve } from "node:path";
+import { createReadStream, createWriteStream, existsSync } from "node:fs";
+import * as zlib from "node:zlib";
+import { gzip } from "node:zlib";
+import { STATUS_CODES } from "node:http";
+import { promisify } from "node:util";
+import { pipeline } from "node:stream/promises";
+//#region src/zip.ts
+const TABLE = (() => {
+	const table = /* @__PURE__ */ new Uint32Array(256);
+	for (let n = 0; n < 256; n += 1) {
+		let c = n;
+		for (let k = 0; k < 8; k += 1) c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
+		table[n] = c >>> 0;
+	}
+	return table;
+})();
+/** This function calculates the CRC-32 of ISO 3309 with a table. It is the fallback of `crc32`. */
+const tableCrc32 = (bytes, value = 0) => {
+	let crc = ~value >>> 0;
+	for (const byte of bytes) crc = TABLE[(crc ^ byte) & 255] ^ crc >>> 8;
+	return ~crc >>> 0;
+};
+const native = zlib.crc32;
+/** This function calculates the CRC-32 of the bytes. It continues from `value`. It uses `zlib.crc32` when Node.js has it. */
+const crc32 = (bytes, value = 0) => native === void 0 ? tableCrc32(bytes, value) : native(bytes, value);
+/** This function writes the whole buffer. A write can write less than the buffer, and the loop writes the rest. */
+const writeAll = async (handle, bytes) => {
+	let at = 0;
+	while (at < bytes.length) {
+		const { bytesWritten } = await handle.write(bytes, at, bytes.length - at);
+		at += bytesWritten;
+	}
+};
+/** A tally writes a new file in sequence. It counts the bytes and calculates their CRC-32 and SHA-256. */
+var TalliedFile = class TalliedFile {
+	path;
+	handle;
+	hash = createHash("sha256");
+	crcValue = 0;
+	count = 0;
+	constructor(path, handle) {
+		this.path = path;
+		this.handle = handle;
+	}
+	/** This method makes a new file. It fails when the file exists. */
+	static async create(path) {
+		return new TalliedFile(path, await open(path, "wx", 384));
+	}
+	/** This value is the count of bytes that the tally wrote. It is the offset of the next write. */
+	get size() {
+		return this.count;
+	}
+	async write(bytes) {
+		await writeAll(this.handle, bytes);
+		this.hash.update(bytes);
+		this.crcValue = crc32(bytes, this.crcValue);
+		this.count += bytes.length;
+	}
+	/** This method closes the file and gives its facts. */
+	async close() {
+		await this.handle.close();
+		return {
+			path: this.path,
+			size: this.count,
+			crc: this.crcValue,
+			sha256: this.hash.digest("hex")
+		};
+	}
+	/** This method closes the file after an error. It gives no facts. */
+	async abandon() {
+		await this.handle.close().catch(() => void 0);
+	}
+};
+/** This function deflates a file into a new file (raw deflate, as ZIP needs it). It gives the size of the new file. */
+const deflateFile = async (from, to) => {
+	await pipeline(createReadStream(from), zlib.createDeflateRaw(), createWriteStream(to, {
+		flags: "wx",
+		mode: 384
+	}));
+	return (await stat(to)).size;
+};
+/** This function makes a member from bytes in memory. A member of method 8 holds the bytes deflated. */
+const memberOf = (name, bytes, method = 8) => {
+	const data = method === 8 ? zlib.deflateRawSync(bytes) : bytes;
+	return {
+		name,
+		method,
+		crc: crc32(bytes),
+		size: bytes.length,
+		compressedSize: data.length,
+		data
+	};
+};
+const MAX32 = 4294967295;
+const MAX16 = 65535;
+const VERSION_ZIP = 20;
+const VERSION_ZIP64 = 45;
+/** The member was made on Unix (3), with version 4.5 of the specification. */
+const MADE_BY = 813;
+/** A plain file that its owner can read and write, and the others can read (`0o100644`). */
+const FILE_MODE = 33188 << 16 >>> 0;
+/** Bit 11 of the flags tells that the name is UTF-8. */
+const UTF8_NAMES = 2048;
+const dosTime = (date) => {
+	const year = Math.min(Math.max(date.getUTCFullYear(), 1980), 2107);
+	return {
+		time: date.getUTCHours() << 11 | date.getUTCMinutes() << 5 | date.getUTCSeconds() >> 1,
+		day: year - 1980 << 9 | date.getUTCMonth() + 1 << 5 | date.getUTCDate()
+	};
+};
+/** This function makes the Zip64 extra field (id 0x0001) with these values of 64 bits. */
+const zip64Extra = (values) => {
+	const extra = Buffer.alloc(4 + 8 * values.length);
+	extra.writeUInt16LE(1, 0);
+	extra.writeUInt16LE(8 * values.length, 2);
+	values.forEach((value, at) => extra.writeBigUInt64LE(BigInt(value), 4 + 8 * at));
+	return extra;
+};
+/**
+* This function writes a ZIP file with these members, in this sequence. It writes a new file and
+* fails when the file exists. It gives the size of the ZIP file.
+*/
+const writeZip = async (target, members, options) => {
+	const always = options.zip64 === "always";
+	const { time, day } = dosTime(options.date);
+	const handle = await open(target, "wx", 384);
+	let offset = 0;
+	const put = async (bytes) => {
+		await writeAll(handle, bytes);
+		offset += bytes.length;
+	};
+	try {
+		const central = [];
+		for (const member of members) {
+			const name = Buffer.from(member.name, "utf8");
+			const start = offset;
+			const bigSizes = always || member.size >= MAX32 || member.compressedSize >= MAX32;
+			const localExtra = bigSizes ? zip64Extra([member.size, member.compressedSize]) : Buffer.alloc(0);
+			const local = Buffer.alloc(30);
+			local.writeUInt32LE(67324752, 0);
+			local.writeUInt16LE(bigSizes ? VERSION_ZIP64 : VERSION_ZIP, 4);
+			local.writeUInt16LE(UTF8_NAMES, 6);
+			local.writeUInt16LE(member.method, 8);
+			local.writeUInt16LE(time, 10);
+			local.writeUInt16LE(day, 12);
+			local.writeUInt32LE(member.crc >>> 0, 14);
+			local.writeUInt32LE(bigSizes ? MAX32 : member.compressedSize, 18);
+			local.writeUInt32LE(bigSizes ? MAX32 : member.size, 22);
+			local.writeUInt16LE(name.length, 26);
+			local.writeUInt16LE(localExtra.length, 28);
+			await put(local);
+			await put(name);
+			await put(localExtra);
+			const before = offset;
+			if (member.file !== void 0) for await (const chunk of createReadStream(member.file)) await put(chunk);
+			else await put(member.data ?? /* @__PURE__ */ new Uint8Array());
+			if (offset - before !== member.compressedSize) throw new Error(`the member ${member.name} has ${String(offset - before)} bytes, not ${String(member.compressedSize)}`);
+			const bigSize = always || member.size >= MAX32;
+			const bigCompressed = always || member.compressedSize >= MAX32;
+			const bigOffset = always || start >= MAX32;
+			const values = [
+				...bigSize ? [member.size] : [],
+				...bigCompressed ? [member.compressedSize] : [],
+				...bigOffset ? [start] : []
+			];
+			const extra = values.length > 0 ? zip64Extra(values) : Buffer.alloc(0);
+			const header = Buffer.alloc(46);
+			header.writeUInt32LE(33639248, 0);
+			header.writeUInt16LE(MADE_BY, 4);
+			header.writeUInt16LE(values.length > 0 ? VERSION_ZIP64 : VERSION_ZIP, 6);
+			header.writeUInt16LE(UTF8_NAMES, 8);
+			header.writeUInt16LE(member.method, 10);
+			header.writeUInt16LE(time, 12);
+			header.writeUInt16LE(day, 14);
+			header.writeUInt32LE(member.crc >>> 0, 16);
+			header.writeUInt32LE(bigCompressed ? MAX32 : member.compressedSize, 20);
+			header.writeUInt32LE(bigSize ? MAX32 : member.size, 24);
+			header.writeUInt16LE(name.length, 28);
+			header.writeUInt16LE(extra.length, 30);
+			header.writeUInt16LE(0, 32);
+			header.writeUInt16LE(0, 34);
+			header.writeUInt16LE(0, 36);
+			header.writeUInt32LE(FILE_MODE, 38);
+			header.writeUInt32LE(bigOffset ? MAX32 : start, 42);
+			central.push(header, name, extra);
+		}
+		const centralStart = offset;
+		for (const part of central) await put(part);
+		const centralSize = offset - centralStart;
+		const count = members.length;
+		const zip64End = always || count >= MAX16 || centralStart >= MAX32 || centralSize >= MAX32;
+		if (zip64End) {
+			const recordAt = offset;
+			const record = Buffer.alloc(56);
+			record.writeUInt32LE(101075792, 0);
+			record.writeBigUInt64LE(44n, 4);
+			record.writeUInt16LE(MADE_BY, 12);
+			record.writeUInt16LE(VERSION_ZIP64, 14);
+			record.writeUInt32LE(0, 16);
+			record.writeUInt32LE(0, 20);
+			record.writeBigUInt64LE(BigInt(count), 24);
+			record.writeBigUInt64LE(BigInt(count), 32);
+			record.writeBigUInt64LE(BigInt(centralSize), 40);
+			record.writeBigUInt64LE(BigInt(centralStart), 48);
+			await put(record);
+			const locator = Buffer.alloc(20);
+			locator.writeUInt32LE(117853008, 0);
+			locator.writeUInt32LE(0, 4);
+			locator.writeBigUInt64LE(BigInt(recordAt), 8);
+			locator.writeUInt32LE(1, 16);
+			await put(locator);
+		}
+		const end = Buffer.alloc(22);
+		end.writeUInt32LE(101010256, 0);
+		end.writeUInt16LE(0, 4);
+		end.writeUInt16LE(0, 6);
+		end.writeUInt16LE(zip64End ? MAX16 : count, 8);
+		end.writeUInt16LE(zip64End ? MAX16 : count, 10);
+		end.writeUInt32LE(zip64End ? MAX32 : centralSize, 12);
+		end.writeUInt32LE(zip64End ? MAX32 : centralStart, 16);
+		end.writeUInt16LE(0, 20);
+		await put(end);
+	} finally {
+		await handle.close();
+	}
+	return offset;
+};
+//#endregion
+//#region src/warc.ts
+var warc_exports = /* @__PURE__ */ __exportAll({
+	base32: () => base32,
+	cdxjLine: () => cdxjLine,
+	exportWarc: () => exportWarc,
+	outputOf: () => outputOf,
+	planArchive: () => planArchive,
+	surt: () => surt,
+	writeArchive: () => writeArchive
+});
+const gzipAsync = promisify(gzip);
+const HOME = new URL("../", BOT_PAGE).href;
+const CRLF = "\r\n";
+const END = Buffer.from("\r\n\r\n", "latin1");
+const KINDS = /* @__PURE__ */ new Set([
+	"capture",
+	"revisit",
+	"skipped"
+]);
+const HASH = /^[0-9a-f]{64}$/;
+/** The revisit records name their original from this count of recent responses. */
+const ORIGINALS = 65536;
+const SERVER_NOT_MODIFIED = "http://netpreserve.org/warc/1.1/revisit/server-not-modified";
+const HTTP_RESPONSE = "application/http; msgtype=response";
+const RFC4648 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
+/** This function writes bytes in base 32 (RFC 4648), with capitals and padding. WARC tools write SHA-256 digests in this form. */
+const base32 = (bytes) => {
+	let out = "";
+	let value = 0;
+	let bits = 0;
+	for (const byte of bytes) {
+		value = (value << 8 | byte) & 65535;
+		bits += 8;
+		while (bits >= 5) {
+			out += RFC4648[value >>> bits - 5 & 31];
+			bits -= 5;
+		}
+	}
+	if (bits > 0) out += RFC4648[value << 5 - bits & 31];
+	while (out.length % 8 !== 0) out += "=";
+	return out;
+};
+const digestOf = (bytes) => `sha256:${base32(bytes)}`;
+const sha256Of = (...parts) => {
+	const hash = createHash("sha256");
+	for (const part of parts) hash.update(part);
+	return hash.digest();
+};
+/** This function gives a version 5 UUID (RFC 9562): the SHA-1 of a namespace and a name. */
+const uuidV5 = (namespace, name) => {
+	const hash = createHash("sha1").update(Buffer.from(namespace.replace(/-/g, ""), "hex")).update(name, "utf8").digest();
+	hash[6] = hash[6] & 15 | 80;
+	hash[8] = hash[8] & 63 | 128;
+	const hex = hash.subarray(0, 16).toString("hex");
+	return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
+};
+/** The namespace of the record ids: a UUID of the URL of the site, in the URL namespace of RFC 9562. */
+const NAMESPACE = uuidV5("6ba7b811-9dad-11d1-80b4-00c04fd430c8", HOME);
+/** The id of a record of a capture is the same at each export: it comes from the hash of the capture. */
+const recordId = (record, type) => `<urn:uuid:${uuidV5(NAMESPACE, `${record.content_hash || `${record.id}@${record.received_at}`}/${type}`)}>`;
+/** A field of a header holds no line break. */
+const clean = (value) => value.replace(/[\r\n]+/g, " ").trim();
+/** This function gives one record: the WARC header, the block and the two line breaks at the end. */
+const recordBytes = (fields, block) => {
+	const length = block.reduce((sum, part) => sum + part.length, 0);
+	const head = [
+		"WARC/1.1",
+		...fields.map(([name, value]) => `${name}: ${clean(value)}`),
+		`Content-Length: ${String(length)}`,
+		"",
+		""
+	].join(CRLF);
+	return Buffer.concat([
+		Buffer.from(head, "utf8"),
+		...block,
+		END
+	]);
+};
+/**
+* This function gives an HTTP status line and header block. HTTP headers are Latin-1 bytes, and
+* undici gives each byte as one character. Thus the block gives back the bytes that the server sent.
+*/
+const httpHead = (status, headers) => Buffer.from([
+	`HTTP/1.1 ${String(status)} ${STATUS_CODES[status] ?? "Unknown"}`,
+	...headers.map(([name, value]) => `${name}: ${clean(value)}`),
+	"",
+	""
+].join(CRLF), "latin1");
+/** These fields of a capture keep a header of the response. */
+const KEPT_HEADERS = [
+	["etag", "ETag"],
+	["last_modified", "Last-Modified"],
+	["cache_control", "Cache-Control"],
+	["expires", "Expires"],
+	["age", "Age"],
+	["date", "Date"],
+	["vary", "Vary"]
+];
+const text = (value) => typeof value === "string" && value.length > 0 ? value : void 0;
+/** This function gives the time of a capture in ISO 8601, with milliseconds. WARC/1.1 permits them. */
+const isoOf = (value, fallback) => {
+	const ms = typeof value === "string" ? Date.parse(value) : NaN;
+	if (Number.isFinite(ms)) return new Date(ms).toISOString();
+	const other = Date.parse(fallback);
+	return Number.isFinite(other) ? new Date(other).toISOString() : fallback;
+};
+const factsOf = (record) => {
+	const payload = record.payload;
+	if (typeof payload !== "object" || payload === null) return void 0;
+	const fields = payload;
+	const url = text(fields["url"]) ?? text(record.external_id);
+	if (url === void 0) return void 0;
+	const blob = text(fields["blob"]);
+	const status = fields["status"];
+	return {
+		url,
+		date: isoOf(fields["fetched_at"], record.received_at),
+		status: typeof status === "number" && Number.isInteger(status) ? status : void 0,
+		blob: blob !== void 0 && HASH.test(blob) ? blob : void 0,
+		contentType: text(fields["content_type"]),
+		rendered: fields["rendered"] === true,
+		truncated: fields["truncated"] === true,
+		cut: fields["cut"] === true,
+		revisitOf: text(fields["revisit_of"]),
+		recipe: text(fields["recipe"]),
+		headers: KEPT_HEADERS.flatMap(([field, name]) => {
+			const value = text(fields[field]);
+			return value === void 0 ? [] : [[name, value]];
+		})
+	};
+};
+const timeOf = (name, value) => {
+	if (value === void 0) return void 0;
+	const ms = value instanceof Date ? value.getTime() : Date.parse(value);
+	if (!Number.isFinite(ms)) throw new RangeError(`${name} is not a time: "${String(value)}"`);
+	return new Date(ms).toISOString();
+};
+const hostName = (value) => {
+	try {
+		return new URL(value).hostname.toLowerCase().replace(/\.$/, "");
+	} catch {
+		return;
+	}
+};
+const filterOf = (options) => {
+	const since = timeOf("since", options.since);
+	const until = timeOf("until", options.until);
+	let host;
+	if (options.host !== void 0) {
+		host = options.host.trim().toLowerCase().replace(/^\*\./, "").replace(/\.$/, "");
+		if (host === "" || /[/:@\s]/.test(host)) throw new RangeError(`host is a host name, for example example.com, not "${options.host}"`);
+	}
+	return {
+		since,
+		until,
+		host,
+		urlPrefix: options.urlPrefix === "" ? void 0 : options.urlPrefix,
+		ids: options.ids === void 0 ? void 0 : new Set(options.ids)
+	};
+};
+const selected = (record, facts, filter) => {
+	if (filter.since !== void 0 && record.received_at < filter.since) return false;
+	if (filter.until !== void 0 && record.received_at >= filter.until) return false;
+	if (filter.urlPrefix !== void 0 && !facts.url.startsWith(filter.urlPrefix)) return false;
+	if (filter.ids !== void 0 && !filter.ids.has(record.id)) return false;
+	if (filter.host !== void 0) {
+		const host = hostName(facts.url);
+		if (host === void 0 || host !== filter.host && !host.endsWith(`.${filter.host}`)) return false;
+	}
+	return true;
+};
+/**
+* This function checks the filters and opens the source. It writes nothing. A folder must hold a
+* capture log, and the export does not make a store.
+*/
+const planArchive = (source, options) => {
+	const filter = filterOf(options);
+	const includeRefusals = options.includeRefusals === true;
+	if (typeof source !== "string") return {
+		captures: source.captures,
+		blobs: source.blobs,
+		filter,
+		includeRefusals
+	};
+	const dir = resolve(source);
+	if (!existsSync(join(dir, "captures"))) throw new Error(`the store ${dir} has no capture log (captures/)`);
+	return {
+		captures: new CaptureLog(join(dir, "captures"), { sync: false }),
+		blobs: new FileBlobStore(join(dir, "blobs"), { sync: false }),
+		filter,
+		includeRefusals
+	};
+};
+/** This function gives the path of the archive and of its temporary file. It fails when the archive exists, unless `overwrite` is true. */
+const outputOf = (options) => {
+	if (typeof options.out !== "string" || options.out.trim() === "") throw new Error("an export needs out: the file of the archive");
+	const target = resolve(options.out);
+	if (options.overwrite !== true && existsSync(target)) throw new Error(`${target} exists: give a different file, or set overwrite`);
+	return {
+		target,
+		temporary: `${target}.${randomUUID()}.tmp`
+	};
+};
+/** This function reads a body from the blob store. It gives no body when the bytes do not agree with the hash. */
+const bodyOf = async (blobs, hash) => {
+	let bytes;
+	try {
+		bytes = await blobs.get(hash);
+	} catch {
+		return;
+	}
+	if (bytes === void 0) return void 0;
+	const digest = sha256Of(bytes);
+	return digest.toString("hex") === hash ? {
+		bytes,
+		digest
+	} : void 0;
+};
+const mimeOf = (contentType) => contentType?.split(";")[0]?.trim().toLowerCase() || "-";
+/** This function gives the JSON of the metadata record of a capture. The labels and the scope of the application stay out. */
+const metadataOf = (record, written) => Buffer.from(`${JSON.stringify({
+	capture: {
+		id: record.id,
+		source: record.source,
+		kind: record.kind,
+		...record.external_id !== void 0 ? { external_id: record.external_id } : {},
+		...record.occurred_at !== void 0 ? { occurred_at: record.occurred_at } : {},
+		received_at: record.received_at,
+		content_hash: record.content_hash,
+		payload: record.payload
+	},
+	export: written
+})}\n`, "utf8");
+/**
+* This function writes the records of an archive into a file: the `warcinfo` record, then the
+* records of each capture that the plan selects. It gives the facts of each `response` and
+* `revisit` record to `onRecord`, for the index of a WACZ file.
+*/
+const writeArchive = async (plan, file, filename, onRecord) => {
+	const counts = {
+		captures: 0,
+		responses: 0,
+		revisits: 0,
+		conversions: 0,
+		metadata: 0,
+		bodiless: 0,
+		missing: 0,
+		damaged: 0
+	};
+	const { filter } = plan;
+	const put = async (bytes) => {
+		const member = await gzipAsync(bytes);
+		const offset = file.size;
+		await file.write(member);
+		return {
+			offset,
+			length: member.length
+		};
+	};
+	const info = `<urn:uuid:${randomUUID()}>`;
+	const shown = Object.fromEntries(Object.entries({
+		since: filter.since,
+		until: filter.until,
+		host: filter.host,
+		url_prefix: filter.urlPrefix,
+		ids: filter.ids === void 0 ? void 0 : filter.ids.size,
+		include_refusals: plan.includeRefusals
+	}).filter(([, value]) => value !== void 0));
+	const fields = [
+		`software: polite-crawler/${VERSION} (+${HOME})`,
+		"format: WARC File Format 1.1",
+		"conformsTo: https://iipc.github.io/warc-specifications/specifications/warc-format/warc-1.1/",
+		"robots: obey",
+		"description: An export of the capture log of polite-crawler. The HTTP headers of each response are rebuilt from the headers that the capture kept.",
+		`polite-crawler-filter: ${JSON.stringify(shown)}`
+	];
+	await put(recordBytes([
+		["WARC-Type", "warcinfo"],
+		["WARC-Record-ID", info],
+		["WARC-Date", (/* @__PURE__ */ new Date()).toISOString()],
+		["WARC-Filename", filename],
+		["Content-Type", "application/warc-fields"]
+	], [Buffer.from(`${fields.join(CRLF)}${CRLF}`, "utf8")]));
+	const originals = /* @__PURE__ */ new Map();
+	const remember = (capture, original) => {
+		originals.delete(capture);
+		originals.set(capture, original);
+		if (originals.size > ORIGINALS) originals.delete(originals.keys().next().value);
+	};
+	const scan = plan.captures.scan({
+		source: "fetch",
+		since: filter.since
+	}, { onDamaged: () => void (counts.damaged += 1) });
+	for await (const record of scan) {
+		if (record.source !== "fetch" || !KINDS.has(record.kind)) continue;
+		const facts = factsOf(record);
+		if (facts === void 0 || !selected(record, facts, filter)) continue;
+		counts.captures += 1;
+		const common = [
+			["WARC-Date", facts.date],
+			["WARC-Target-URI", facts.url],
+			["WARC-Warcinfo-ID", info]
+		];
+		const vendor = [["WARC-Polite-Crawler-Capture", record.id]];
+		let main;
+		let written = {
+			record: "none",
+			reason: "no body"
+		};
+		if (record.kind === "revisit" && facts.blob !== void 0) {
+			const original = facts.revisitOf === void 0 ? void 0 : originals.get(facts.revisitOf);
+			const head = httpHead(304, [...facts.contentType !== void 0 ? [["Content-Type", facts.contentType]] : [], ...facts.headers]);
+			const digest = digestOf(Buffer.from(facts.blob, "hex"));
+			main = recordId(record, "revisit");
+			const at = await put(recordBytes([
+				["WARC-Type", "revisit"],
+				["WARC-Record-ID", main],
+				...common,
+				["WARC-Profile", SERVER_NOT_MODIFIED],
+				...original !== void 0 ? [["WARC-Refers-To", original.id]] : [],
+				["WARC-Refers-To-Target-URI", original?.url ?? facts.url],
+				...original !== void 0 ? [["WARC-Refers-To-Date", original.date]] : [],
+				["WARC-Payload-Digest", digest],
+				["WARC-Block-Digest", digestOf(sha256Of(head))],
+				...vendor,
+				["WARC-Polite-Crawler-Headers", "rebuilt"],
+				["Content-Type", HTTP_RESPONSE]
+			], [head]));
+			counts.revisits += 1;
+			written = {
+				record: "revisit",
+				http_headers: "rebuilt"
+			};
+			await onRecord?.({
+				type: "revisit",
+				url: facts.url,
+				date: facts.date,
+				status: 304,
+				mime: "warc/revisit",
+				digest,
+				...at,
+				capture: record.id,
+				page: false
+			});
+		} else if (record.kind === "capture" && facts.blob !== void 0) {
+			const body = await bodyOf(plan.blobs, facts.blob);
+			if (body === void 0) {
+				counts.missing += 1;
+				written = {
+					record: "none",
+					reason: "body missing"
+				};
+			} else if (facts.rendered) {
+				const digest = digestOf(body.digest);
+				main = recordId(record, "conversion");
+				await put(recordBytes([
+					["WARC-Type", "conversion"],
+					["WARC-Record-ID", main],
+					...common,
+					["WARC-Payload-Digest", digest],
+					["WARC-Block-Digest", digest],
+					...vendor,
+					["Content-Type", facts.contentType ?? "text/html; charset=utf-8"]
+				], [body.bytes]));
+				counts.conversions += 1;
+				written = {
+					record: "conversion",
+					body: "rendered"
+				};
+			} else if (facts.status !== void 0 && facts.status >= 100 && facts.status <= 999) {
+				const head = httpHead(facts.status, [
+					...facts.contentType !== void 0 ? [["Content-Type", facts.contentType]] : [],
+					["Content-Length", String(body.bytes.length)],
+					...facts.headers
+				]);
+				const digest = digestOf(body.digest);
+				main = recordId(record, "response");
+				const truncated = facts.cut ? "disconnect" : facts.truncated ? "length" : void 0;
+				const at = await put(recordBytes([
+					["WARC-Type", "response"],
+					["WARC-Record-ID", main],
+					...common,
+					["WARC-Payload-Digest", digest],
+					["WARC-Block-Digest", digestOf(sha256Of(head, body.bytes))],
+					...truncated !== void 0 ? [["WARC-Truncated", truncated]] : [],
+					...vendor,
+					["WARC-Polite-Crawler-Headers", "rebuilt"],
+					["Content-Type", HTTP_RESPONSE]
+				], [head, body.bytes]));
+				counts.responses += 1;
+				written = {
+					record: "response",
+					http_headers: "rebuilt",
+					body: "decoded"
+				};
+				remember(record.id, {
+					id: main,
+					date: facts.date,
+					url: facts.url
+				});
+				await onRecord?.({
+					type: "response",
+					url: facts.url,
+					date: facts.date,
+					status: facts.status,
+					mime: mimeOf(facts.contentType),
+					digest,
+					...at,
+					capture: record.id,
+					page: facts.status === 200 && facts.recipe !== "robots.txt"
+				});
+			}
+		}
+		if (main === void 0) {
+			if (written["reason"] === "no body") counts.bodiless += 1;
+			if (!plan.includeRefusals) continue;
+		}
+		const block = metadataOf(record, written);
+		await put(recordBytes([
+			["WARC-Type", "metadata"],
+			["WARC-Record-ID", recordId(record, "metadata")],
+			...common,
+			...main !== void 0 ? [["WARC-Concurrent-To", main]] : [],
+			["WARC-Block-Digest", digestOf(sha256Of(block))],
+			...vendor,
+			["Content-Type", "application/json"]
+		], [block]));
+		counts.metadata += 1;
+	}
+	return counts;
+};
+/**
+* This function gives the SURT key of a URL, as `pywb` and the `surt` library make it with their
+* defaults. The host is reversed with commas and has no `www.` and no default port. The path
+* and the query are lowercase, the path has no slash at the end, and the parts of the query are
+* sorted.
+*/
+const surt = (value) => {
+	let url;
+	try {
+		url = new URL(value);
+	} catch {
+		return value.toLowerCase();
+	}
+	const host = url.hostname.toLowerCase().replace(/\.$/, "").replace(/^www\d*\./, "");
+	const key = host.startsWith("[") ? host : host.split(".").reverse().join(",");
+	let path = url.pathname.toLowerCase();
+	if (path.length > 1 && path.endsWith("/")) path = path.slice(0, -1);
+	const query = url.search.slice(1).toLowerCase().split("&").filter(Boolean).sort().join("&");
+	return `${key}${url.port === "" ? "" : `:${url.port}`})${path}${query === "" ? "" : `?${query}`}`;
+};
+/** This function gives the line of a record in a CDXJ index: the SURT key, the time to the second, and JSON. */
+const cdxjLine = (record, filename) => `${surt(record.url)} ${record.date.slice(0, 19).replace(/\D/g, "")} ${JSON.stringify({
+	url: record.url,
+	mime: record.mime,
+	status: String(record.status),
+	digest: record.digest,
+	length: String(record.length),
+	offset: String(record.offset),
+	filename
+})}`;
+/**
+* This function writes the captures of a store as a WARC/1.1 file, with one gzip member for each
+* record (`.warc.gz`). It reads the capture log as a stream, so its memory does not grow with the
+* store. It writes a temporary file and gives it the name `out` at the end, so a failed export
+* leaves no part of a file.
+*
+* The source is the folder of a store, or the capture log and the blob store of a runtime
+* (`crawler.runtime`). The export only reads the store.
+*/
+const exportWarc = async (source, options) => {
+	const plan = planArchive(source, options);
+	const { target, temporary } = outputOf(options);
+	const file = await TalliedFile.create(temporary);
+	try {
+		const counts = await writeArchive(plan, file, basename(target));
+		const done = await file.close();
+		await rename(temporary, target);
+		return {
+			format: "warc",
+			out: target,
+			bytes: done.size,
+			...counts
+		};
+	} catch (error) {
+		await file.abandon();
+		await rm(temporary, { force: true }).catch(() => void 0);
+		throw error;
+	}
+};
+//#endregion
+export { warc_exports as a, deflateFile as c, planArchive as i, memberOf as l, exportWarc as n, writeArchive as o, outputOf as r, TalliedFile as s, cdxjLine as t, writeZip as u };
+
+//# sourceMappingURL=warc-VIPscWxY.mjs.map