polite-crawler 0.1.1 → 0.1.2

package/build/urls-BHFeSyEK.mjs

@@ -0,0 +1,62 @@
+//#region src/urls.ts
+/**
+* This module gives the canonical form of a URL. The document id of a page and the cache key of the
+* crawler use it. Thus two spellings of one page are one document and one cached answer.
+*/
+const TRACKING = /^(utm_[a-z_]+|fbclid|gclid|gclsrc|dclid|gbraid|wbraid|msclkid|yclid|ttclid|twclid|li_fat_id|srsltid|mc_[a-z]+|_ga|_gl|_hsenc|_hsmi|__hstc|__hssc|__hsfp|mkt_tok|ref_src|igshid)$/i;
+/** This function decodes the escapes of unreserved characters (`%7E` is `~`), and writes the other escapes in capitals. */
+const normalizeEscapes = (value) => value.replace(/%[0-9a-f]{2}/gi, (escape) => {
+	const char = String.fromCharCode(Number.parseInt(escape.slice(1), 16));
+	return /^[A-Za-z0-9\-._~]$/.test(char) ? char : escape.toUpperCase();
+});
+/**
+* This function gives one URL for each page: a lowercase host, and no fragment, tracking
+* parameters, empty query or default port. It decodes the escapes of unreserved characters.
+*/
+const canonicalUrl = (value) => {
+	const url = new URL(value);
+	url.hash = "";
+	url.hostname = url.hostname.toLowerCase();
+	if (url.protocol === "https:" && url.port === "443" || url.protocol === "http:" && url.port === "80") url.port = "";
+	for (const key of Array.from(url.searchParams.keys())) if (TRACKING.test(key)) url.searchParams.delete(key);
+	if (url.pathname.includes("%")) url.pathname = normalizeEscapes(url.pathname);
+	url.search = url.search.includes("%") ? normalizeEscapes(url.search) : url.search;
+	return url.href;
+};
+/**
+* This function gives the cache key of a URL: its canonical form. A text that is not a URL stays as
+* it is. The crawler and `reindex` use the same key, so a rebuilt index answers the same requests.
+*/
+const cacheKeyOf = (value) => {
+	try {
+		return canonicalUrl(value);
+	} catch {
+		return value;
+	}
+};
+/**
+* This function gives the site of a host name: the name in lowercase, with no final dot and no
+* `www.` at the start. Thus `www.example.com` and `example.com` are one site.
+*/
+const siteOf = (host) => host.toLowerCase().replace(/\.$/, "").replace(/^www\./, "");
+/**
+* This function gives the test of the scope of a crawl or a discovery. With `sameSite: false`
+* and no `allowHosts`, each http(s) URL is in scope.
+*/
+const scopeTest = (starts, options) => {
+	const sameSite = options.sameSite !== false;
+	const sites = new Set(starts.map((start) => siteOf(start.hostname)));
+	const hosts = (options.allowHosts ?? []).map((host) => host.trim().toLowerCase().replace(/\.$/, ""));
+	const any = !sameSite && options.allowHosts === void 0;
+	return (url) => {
+		if (url.protocol !== "http:" && url.protocol !== "https:") return false;
+		if (any) return true;
+		const name = url.hostname.toLowerCase().replace(/\.$/, "");
+		if (sameSite && sites.has(siteOf(name))) return true;
+		return hosts.some((host) => host.startsWith("*.") ? name.endsWith(host.slice(1)) : name === host);
+	};
+};
+//#endregion
+export { canonicalUrl as n, scopeTest as r, cacheKeyOf as t };
+
+//# sourceMappingURL=urls-BHFeSyEK.mjs.map

package/build/urls-BHFeSyEK.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"urls-BHFeSyEK.mjs","names":[],"sources":["../src/urls.ts"],"sourcesContent":["/**\n * This module gives the canonical form of a URL. The document id of a page and the cache key of the\n * crawler use it. Thus two spellings of one page are one document and one cached answer.\n */\n\nconst TRACKING =\n  /^(utm_[a-z_]+|fbclid|gclid|gclsrc|dclid|gbraid|wbraid|msclkid|yclid|ttclid|twclid|li_fat_id|srsltid|mc_[a-z]+|_ga|_gl|_hsenc|_hsmi|__hstc|__hssc|__hsfp|mkt_tok|ref_src|igshid)$/i;\n\n/** This function decodes the escapes of unreserved characters (`%7E` is `~`), and writes the other escapes in capitals. */\nconst normalizeEscapes = (value: string): string =>\n  value.replace(/%[0-9a-f]{2}/gi, (escape) => {\n    const char = String.fromCharCode(Number.parseInt(escape.slice(1), 16));\n    return /^[A-Za-z0-9\\-._~]$/.test(char) ? char : escape.toUpperCase();\n  });\n\n/**\n * This function gives one URL for each page: a lowercase host, and no fragment, tracking\n * parameters, empty query or default port. It decodes the escapes of unreserved characters.\n */\nexport const canonicalUrl = (value: string): string => {\n  const url = new URL(value);\n  url.hash = \"\";\n  url.hostname = url.hostname.toLowerCase();\n  if (\n    (url.protocol === \"https:\" && url.port === \"443\") ||\n    (url.protocol === \"http:\" && url.port === \"80\")\n  ) {\n    url.port = \"\";\n  }\n  // a copy of the keys, because the loop deletes from the live list\n  for (const key of Array.from(url.searchParams.keys())) {\n    if (TRACKING.test(key)) url.searchParams.delete(key);\n  }\n  if (url.pathname.includes(\"%\")) url.pathname = normalizeEscapes(url.pathname);\n  // \"/p?\" is \"/p\"\n  url.search = url.search.includes(\"%\") ? normalizeEscapes(url.search) : url.search;\n  return url.href;\n};\n\n/**\n * This function gives the cache key of a URL: its canonical form. A text that is not a URL stays as\n * it is. The crawler and `reindex` use the same key, so a rebuilt index answers the same requests.\n */\nexport const cacheKeyOf = (value: string): string => {\n  try {\n    return canonicalUrl(value);\n  } catch {\n    return value;\n  }\n};\n\n/**\n * This function gives the site of a host name: the name in lowercase, with no final dot and no\n * `www.` at the start. Thus `www.example.com` and `example.com` are one site.\n */\nexport const siteOf = (host: string): string =>\n  host\n    .toLowerCase()\n    .replace(/\\.$/, \"\")\n    .replace(/^www\\./, \"\");\n\n/** These options set the hosts that a crawl or a discovery keeps. */\nexport interface ScopeOptions {\n  /** With the default value true, the URLs on the sites of the start URLs are in scope. */\n  readonly sameSite?: boolean | undefined;\n  /** The URLs on these hosts are in scope too. A name that starts with `*.` names the hosts under it. */\n  readonly allowHosts?: readonly string[] | undefined;\n}\n\n/**\n * This function gives the test of the scope of a crawl or a discovery. With `sameSite: false`\n * and no `allowHosts`, each http(s) URL is in scope.\n */\nexport const scopeTest = (\n  starts: readonly URL[],\n  options: ScopeOptions,\n): ((url: URL) => boolean) => {\n  const sameSite = options.sameSite !== false;\n  const sites = new Set(starts.map((start) => siteOf(start.hostname)));\n  const hosts = (options.allowHosts ?? []).map((host) =>\n    host.trim().toLowerCase().replace(/\\.$/, \"\"),\n  );\n  const any = !sameSite && options.allowHosts === undefined;\n  return (url) => {\n    if (url.protocol !== \"http:\" && url.protocol !== \"https:\") return false;\n    if (any) return true;\n    const name = url.hostname.toLowerCase().replace(/\\.$/, \"\");\n    if (sameSite && sites.has(siteOf(name))) return true;\n    return hosts.some((host) =>\n      host.startsWith(\"*.\") ? name.endsWith(host.slice(1)) : name === host,\n    );\n  };\n};\n"],"mappings":";;;;;AAKA,MAAM,WACJ;;AAGF,MAAM,oBAAoB,UACxB,MAAM,QAAQ,mBAAmB,WAAW;CAC1C,MAAM,OAAO,OAAO,aAAa,OAAO,SAAS,OAAO,MAAM,CAAC,GAAG,EAAE,CAAC;CACrE,OAAO,qBAAqB,KAAK,IAAI,IAAI,OAAO,OAAO,YAAY;AACrE,CAAC;;;;;AAMH,MAAa,gBAAgB,UAA0B;CACrD,MAAM,MAAM,IAAI,IAAI,KAAK;CACzB,IAAI,OAAO;CACX,IAAI,WAAW,IAAI,SAAS,YAAY;CACxC,IACG,IAAI,aAAa,YAAY,IAAI,SAAS,SAC1C,IAAI,aAAa,WAAW,IAAI,SAAS,MAE1C,IAAI,OAAO;CAGb,KAAK,MAAM,OAAO,MAAM,KAAK,IAAI,aAAa,KAAK,CAAC,GAClD,IAAI,SAAS,KAAK,GAAG,GAAG,IAAI,aAAa,OAAO,GAAG;CAErD,IAAI,IAAI,SAAS,SAAS,GAAG,GAAG,IAAI,WAAW,iBAAiB,IAAI,QAAQ;CAE5E,IAAI,SAAS,IAAI,OAAO,SAAS,GAAG,IAAI,iBAAiB,IAAI,MAAM,IAAI,IAAI;CAC3E,OAAO,IAAI;AACb;;;;;AAMA,MAAa,cAAc,UAA0B;CACnD,IAAI;EACF,OAAO,aAAa,KAAK;CAC3B,QAAQ;EACN,OAAO;CACT;AACF;;;;;AAMA,MAAa,UAAU,SACrB,KACG,YAAY,CAAC,CACb,QAAQ,OAAO,EAAE,CAAC,CAClB,QAAQ,UAAU,EAAE;;;;;AAczB,MAAa,aACX,QACA,YAC4B;CAC5B,MAAM,WAAW,QAAQ,aAAa;CACtC,MAAM,QAAQ,IAAI,IAAI,OAAO,KAAK,UAAU,OAAO,MAAM,QAAQ,CAAC,CAAC;CACnE,MAAM,SAAS,QAAQ,cAAc,CAAC,EAAA,CAAG,KAAK,SAC5C,KAAK,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE,CAC7C;CACA,MAAM,MAAM,CAAC,YAAY,QAAQ,eAAe,KAAA;CAChD,QAAQ,QAAQ;EACd,IAAI,IAAI,aAAa,WAAW,IAAI,aAAa,UAAU,OAAO;EAClE,IAAI,KAAK,OAAO;EAChB,MAAM,OAAO,IAAI,SAAS,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE;EACzD,IAAI,YAAY,MAAM,IAAI,OAAO,IAAI,CAAC,GAAG,OAAO;EAChD,OAAO,MAAM,MAAM,SACjB,KAAK,WAAW,IAAI,IAAI,KAAK,SAAS,KAAK,MAAM,CAAC,CAAC,IAAI,SAAS,IAClE;CACF;AACF"}

package/build/urls-BTDSgDLC.d.mts

@@ -0,0 +1,16 @@
+//#region src/urls.d.ts
+/**
+ * This function gives one URL for each page: a lowercase host, and no fragment, tracking
+ * parameters, empty query or default port. It decodes the escapes of unreserved characters.
+ */
+declare const canonicalUrl: (value: string) => string;
+/** These options set the hosts that a crawl or a discovery keeps. */
+interface ScopeOptions {
+  /** With the default value true, the URLs on the sites of the start URLs are in scope. */
+  readonly sameSite?: boolean | undefined;
+  /** The URLs on these hosts are in scope too. A name that starts with `*.` names the hosts under it. */
+  readonly allowHosts?: readonly string[] | undefined;
+}
+//#endregion
+export { canonicalUrl as n, ScopeOptions as t };
+//# sourceMappingURL=urls-BTDSgDLC.d.mts.map

package/build/wacz-sfOLT8e9.mjs

@@ -0,0 +1,275 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { i as VERSION } from "./duration-iIuMMofi.mjs";
+import { t as cacheKeyOf } from "./urls-BHFeSyEK.mjs";
+import { c as deflateFile, i as planArchive, l as memberOf, o as writeArchive, r as outputOf, s as TalliedFile, t as cdxjLine, u as writeZip } from "./warc-VIPscWxY.mjs";
+import { createHash } from "node:crypto";
+import { mkdir, rename, rm, writeFile } from "node:fs/promises";
+import { basename, join } from "node:path";
+import { createReadStream } from "node:fs";
+//#region src/sorter.ts
+const NEWLINE$1 = 10;
+/** This function compares two lines by their bytes. */
+const byBytes = (a, b) => Buffer.compare(a, b);
+/** This function reads a file line by line, as bytes. A line holds no line break. */
+async function* linesOf(path) {
+	let rest;
+	for await (const chunk of createReadStream(path)) {
+		const data = rest === void 0 ? chunk : Buffer.concat([rest, chunk]);
+		rest = void 0;
+		let start = 0;
+		for (;;) {
+			const end = data.indexOf(NEWLINE$1, start);
+			if (end === -1) break;
+			yield data.subarray(start, end);
+			start = end + 1;
+		}
+		if (start < data.length) rest = Buffer.from(data.subarray(start));
+	}
+	if (rest !== void 0 && rest.length > 0) yield rest;
+}
+/**
+* A line sorter sorts lines of text in the sequence of their UTF-8 bytes (the sequence of
+* `LC_ALL=C sort`). It keeps no more than `limit` bytes of lines in memory. When it has more, it
+* writes them to a sorted file in `dir`. At the end it merges the files. Thus its memory does not
+* grow with the count of lines. The WACZ export sorts its index and its page list with it.
+*
+* A line must not hold a line break.
+*/
+var LineSorter = class {
+	dir;
+	limit;
+	lines = [];
+	bytes = 0;
+	runs = [];
+	constructor(dir, limit = 8388608) {
+		this.dir = dir;
+		this.limit = limit;
+	}
+	/** This value is the count of sorted files that the sorter wrote. */
+	get files() {
+		return this.runs.length;
+	}
+	async add(line) {
+		const bytes = Buffer.from(line, "utf8");
+		if (bytes.includes(NEWLINE$1)) throw new Error("a line to sort holds a line break");
+		this.lines.push(bytes);
+		this.bytes += bytes.length + 64;
+		if (this.bytes >= this.limit) await this.spill();
+	}
+	async spill() {
+		if (this.lines.length === 0) return;
+		this.lines.sort(byBytes);
+		await mkdir(this.dir, { recursive: true });
+		const path = join(this.dir, `${String(this.runs.length)}.lines`);
+		const parts = [];
+		const newline = Buffer.from([NEWLINE$1]);
+		for (const line of this.lines) parts.push(line, newline);
+		await writeFile(path, Buffer.concat(parts), { mode: 384 });
+		this.runs.push(path);
+		this.lines = [];
+		this.bytes = 0;
+	}
+	/** This method gives each line in the sorted sequence. Use it one time, after the last `add`. */
+	async *sorted() {
+		if (this.runs.length === 0) {
+			const lines = this.lines.sort(byBytes);
+			this.lines = [];
+			yield* lines;
+			return;
+		}
+		await this.spill();
+		const readers = this.runs.map((path) => linesOf(path)[Symbol.asyncIterator]());
+		const heads = [];
+		for (const reader of readers) heads.push((await reader.next()).value ?? void 0);
+		for (;;) {
+			let pick = -1;
+			for (const [at, head] of heads.entries()) if (head !== void 0 && (pick === -1 || Buffer.compare(head, heads[pick]) < 0)) pick = at;
+			if (pick === -1) return;
+			yield heads[pick];
+			heads[pick] = (await readers[pick].next()).value ?? void 0;
+		}
+	}
+};
+//#endregion
+//#region src/wacz.ts
+var wacz_exports = /* @__PURE__ */ __exportAll({ exportWacz: () => exportWacz });
+/** This is the version of the WACZ specification that the export follows. */
+const WACZ_VERSION = "1.1.1";
+const WARC_NAME = "data.warc.gz";
+const NEWLINE = Buffer.from("\n");
+const TAB = 9;
+const PAGES_HEADER = {
+	format: "json-pages-1.0",
+	id: "pages",
+	title: "All Pages"
+};
+/** This function writes lines to a file in batches of approximately 64 KiB. It gives the count of lines. */
+const writeLines = async (file, lines) => {
+	let batch = [];
+	let size = 0;
+	let count = 0;
+	for await (const line of lines) {
+		batch.push(line, NEWLINE);
+		size += line.length + 1;
+		count += 1;
+		if (size >= 65536) {
+			await file.write(Buffer.concat(batch));
+			batch = [];
+			size = 0;
+		}
+	}
+	if (batch.length > 0) await file.write(Buffer.concat(batch));
+	return count;
+};
+/**
+* This function keeps the last line of each document. Each line is the key of the document, a tab,
+* the time, a tab and the JSON of the page. The sorter gives the lines of one key together, the
+* oldest first.
+*/
+async function* latestOf(lines) {
+	let held;
+	let heldKey;
+	for await (const line of lines) {
+		const key = line.subarray(0, line.indexOf(TAB));
+		if (held !== void 0 && heldKey !== void 0 && !key.equals(heldKey)) yield held.subarray(held.indexOf(TAB, heldKey.length + 1) + 1);
+		held = line;
+		heldKey = key;
+	}
+	if (held !== void 0 && heldKey !== void 0) yield held.subarray(held.indexOf(TAB, heldKey.length + 1) + 1);
+}
+const resourceOf = (path, facts) => ({
+	name: basename(path).toLowerCase(),
+	path,
+	hash: `sha256:${facts.sha256}`,
+	bytes: facts.size
+});
+/**
+* This function writes the captures of a store as a WACZ 1.1.1 file. The ZIP holds these parts:
+*
+* - `archive/data.warc.gz`: the WARC file of `exportWarc`, stored with no more compression, so a
+*   replay tool reads a record at its offset.
+* - `indexes/index.cdxj`: the index of the `response` and `revisit` records, sorted by SURT key.
+* - `pages/pages.jsonl`: one page for each document, at its latest capture.
+* - `datapackage.json` and `datapackage-digest.json`: the SHA-256 of each part, and of the package.
+*
+* The index and the page list are sorted on the disk, so the memory does not grow with the store.
+* The export writes temporary files beside `out` and removes them at the end.
+*/
+const exportWacz = async (source, options) => {
+	const plan = planArchive(source, options);
+	const { target, temporary } = outputOf(options);
+	const parts = `${temporary}.parts`;
+	await mkdir(parts, {
+		recursive: true,
+		mode: 448
+	});
+	try {
+		const created = /* @__PURE__ */ new Date();
+		const warc = await TalliedFile.create(join(parts, WARC_NAME));
+		const index = new LineSorter(join(parts, "index"));
+		const pages = new LineSorter(join(parts, "pages"));
+		let counts;
+		try {
+			counts = await writeArchive(plan, warc, WARC_NAME, async (record) => {
+				await index.add(cdxjLine(record, WARC_NAME));
+				if (record.page) {
+					const page = JSON.stringify({
+						id: record.capture,
+						url: record.url,
+						ts: record.date
+					});
+					await pages.add(`${cacheKeyOf(record.url)}\t${record.date}\t${page}`);
+				}
+			});
+		} catch (error) {
+			await warc.abandon();
+			throw error;
+		}
+		const archive = await warc.close();
+		const indexFile = await TalliedFile.create(join(parts, "index.cdxj"));
+		try {
+			await writeLines(indexFile, index.sorted());
+		} catch (error) {
+			await indexFile.abandon();
+			throw error;
+		}
+		const indexFacts = await indexFile.close();
+		const pagesFile = await TalliedFile.create(join(parts, "pages.jsonl"));
+		let pageCount;
+		try {
+			await pagesFile.write(Buffer.from(`${JSON.stringify(PAGES_HEADER)}\n`));
+			pageCount = await writeLines(pagesFile, latestOf(pages.sorted()));
+		} catch (error) {
+			await pagesFile.abandon();
+			throw error;
+		}
+		const pagesFacts = await pagesFile.close();
+		const datapackage = Buffer.from(JSON.stringify({
+			profile: "data-package",
+			wacz_version: WACZ_VERSION,
+			software: `polite-crawler ${VERSION}`,
+			created: created.toISOString().replace(/\.\d{3}Z$/, "Z"),
+			title: "polite-crawler export",
+			resources: [
+				resourceOf(`archive/${WARC_NAME}`, archive),
+				resourceOf("indexes/index.cdxj", indexFacts),
+				resourceOf("pages/pages.jsonl", pagesFacts)
+			]
+		}, null, 2), "utf8");
+		const digest = Buffer.from(JSON.stringify({
+			path: "datapackage.json",
+			hash: `sha256:${createHash("sha256").update(datapackage).digest("hex")}`
+		}, null, 2), "utf8");
+		const deflated = async (name, facts) => {
+			const file = `${facts.path}.deflate`;
+			return {
+				name,
+				method: 8,
+				crc: facts.crc,
+				size: facts.size,
+				compressedSize: await deflateFile(facts.path, file),
+				file
+			};
+		};
+		const members = [
+			{
+				name: `archive/${WARC_NAME}`,
+				method: 0,
+				crc: archive.crc,
+				size: archive.size,
+				compressedSize: archive.size,
+				file: archive.path
+			},
+			await deflated("indexes/index.cdxj", indexFacts),
+			await deflated("pages/pages.jsonl", pagesFacts),
+			memberOf("datapackage.json", datapackage),
+			memberOf("datapackage-digest.json", digest)
+		];
+		let bytes;
+		try {
+			bytes = await writeZip(temporary, members, { date: created });
+			await rename(temporary, target);
+		} catch (error) {
+			await rm(temporary, { force: true }).catch(() => void 0);
+			throw error;
+		}
+		return {
+			format: "wacz",
+			out: target,
+			bytes,
+			pages: pageCount,
+			...counts
+		};
+	} finally {
+		await rm(parts, {
+			recursive: true,
+			force: true,
+			maxRetries: 5,
+			retryDelay: 100
+		});
+	}
+};
+//#endregion
+export { wacz_exports as n, exportWacz as t };
+
+//# sourceMappingURL=wacz-sfOLT8e9.mjs.map

package/build/wacz-sfOLT8e9.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"wacz-sfOLT8e9.mjs","names":["NEWLINE"],"sources":["../src/sorter.ts","../src/wacz.ts"],"sourcesContent":["import { createReadStream } from \"node:fs\";\nimport { mkdir, writeFile } from \"node:fs/promises\";\nimport { join } from \"node:path\";\n\nconst NEWLINE = 0x0a;\n\n/** This function compares two lines by their bytes. */\nconst byBytes = (a: Buffer, b: Buffer): number => Buffer.compare(a, b);\n\n/** This function reads a file line by line, as bytes. A line holds no line break. */\nasync function* linesOf(path: string): AsyncGenerator<Buffer> {\n  let rest: Buffer | undefined;\n  for await (const chunk of createReadStream(path) as AsyncIterable<Buffer>) {\n    const data = rest === undefined ? chunk : Buffer.concat([rest, chunk]);\n    rest = undefined;\n    let start = 0;\n    for (;;) {\n      const end = data.indexOf(NEWLINE, start);\n      if (end === -1) break;\n      yield data.subarray(start, end);\n      start = end + 1;\n    }\n    if (start < data.length) rest = Buffer.from(data.subarray(start));\n  }\n  if (rest !== undefined && rest.length > 0) yield rest;\n}\n\n/**\n * A line sorter sorts lines of text in the sequence of their UTF-8 bytes (the sequence of\n * `LC_ALL=C sort`). It keeps no more than `limit` bytes of lines in memory. When it has more, it\n * writes them to a sorted file in `dir`. At the end it merges the files. Thus its memory does not\n * grow with the count of lines. The WACZ export sorts its index and its page list with it.\n *\n * A line must not hold a line break.\n */\nexport class LineSorter {\n  private lines: Buffer[] = [];\n  private bytes = 0;\n  private readonly runs: string[] = [];\n\n  constructor(\n    private readonly dir: string,\n    private readonly limit: number = 8 * 1024 * 1024,\n  ) {}\n\n  /** This value is the count of sorted files that the sorter wrote. */\n  get files(): number {\n    return this.runs.length;\n  }\n\n  async add(line: string): Promise<void> {\n    const bytes = Buffer.from(line, \"utf8\");\n    if (bytes.includes(NEWLINE)) throw new Error(\"a line to sort holds a line break\");\n    this.lines.push(bytes);\n    // each line also costs an object in memory, so the count adds a little for each line\n    this.bytes += bytes.length + 64;\n    if (this.bytes >= this.limit) await this.spill();\n  }\n\n  private async spill(): Promise<void> {\n    if (this.lines.length === 0) return;\n    this.lines.sort(byBytes);\n    await mkdir(this.dir, { recursive: true });\n    const path = join(this.dir, `${String(this.runs.length)}.lines`);\n    const parts: Buffer[] = [];\n    const newline = Buffer.from([NEWLINE]);\n    for (const line of this.lines) parts.push(line, newline);\n    await writeFile(path, Buffer.concat(parts), { mode: 0o600 });\n    this.runs.push(path);\n    this.lines = [];\n    this.bytes = 0;\n  }\n\n  /** This method gives each line in the sorted sequence. Use it one time, after the last `add`. */\n  async *sorted(): AsyncGenerator<Buffer> {\n    if (this.runs.length === 0) {\n      const lines = this.lines.sort(byBytes);\n      this.lines = [];\n      yield* lines;\n      return;\n    }\n    await this.spill();\n    const readers = this.runs.map((path) => linesOf(path)[Symbol.asyncIterator]());\n    const heads: Array<Buffer | undefined> = [];\n    for (const reader of readers) heads.push((await reader.next()).value ?? undefined);\n    for (;;) {\n      let pick = -1;\n      for (const [at, head] of heads.entries()) {\n        if (head !== undefined && (pick === -1 || Buffer.compare(head, heads[pick]!) < 0))\n          pick = at;\n      }\n      if (pick === -1) return;\n      yield heads[pick]!;\n      heads[pick] = (await readers[pick]!.next()).value ?? undefined;\n    }\n  }\n}\n","import { createHash } from \"node:crypto\";\nimport { mkdir, rename, rm } from \"node:fs/promises\";\nimport { basename, join } from \"node:path\";\nimport { LineSorter } from \"./sorter.ts\";\nimport { cacheKeyOf } from \"./urls.ts\";\nimport { VERSION } from \"./version.ts\";\nimport type { ArchiveCounts, ArchiveOptions, ArchiveSource } from \"./warc.ts\";\nimport { cdxjLine, outputOf, planArchive, writeArchive } from \"./warc.ts\";\nimport type { Tallied, ZipMember } from \"./zip.ts\";\nimport { TalliedFile, deflateFile, memberOf, writeZip } from \"./zip.ts\";\n\n/*\n * This module writes a WACZ 1.1.1 file: a ZIP with the WARC file, a sorted CDXJ index, a page list\n * and the data package with the SHA-256 of each part. Replay tools (ReplayWeb.page, pywb) open it.\n */\n\n/** This is the version of the WACZ specification that the export follows. */\nconst WACZ_VERSION = \"1.1.1\";\nconst WARC_NAME = \"data.warc.gz\";\nconst NEWLINE = Buffer.from(\"\\n\");\nconst TAB = 0x09;\nconst PAGES_HEADER = { format: \"json-pages-1.0\", id: \"pages\", title: \"All Pages\" };\n\n/** This is the result of `exportWacz`. */\nexport interface WaczResult extends ArchiveCounts {\n  readonly format: \"wacz\";\n  /** This value is the absolute path of the archive. */\n  readonly out: string;\n  readonly bytes: number;\n  /** This value is the count of pages in `pages/pages.jsonl`: one for each document. */\n  readonly pages: number;\n}\n\n/** This function writes lines to a file in batches of approximately 64 KiB. It gives the count of lines. */\nconst writeLines = async (file: TalliedFile, lines: AsyncIterable<Uint8Array>): Promise<number> => {\n  let batch: Uint8Array[] = [];\n  let size = 0;\n  let count = 0;\n  for await (const line of lines) {\n    batch.push(line, NEWLINE);\n    size += line.length + 1;\n    count += 1;\n    if (size >= 65_536) {\n      await file.write(Buffer.concat(batch));\n      batch = [];\n      size = 0;\n    }\n  }\n  if (batch.length > 0) await file.write(Buffer.concat(batch));\n  return count;\n};\n\n/**\n * This function keeps the last line of each document. Each line is the key of the document, a tab,\n * the time, a tab and the JSON of the page. The sorter gives the lines of one key together, the\n * oldest first.\n */\nasync function* latestOf(lines: AsyncIterable<Buffer>): AsyncGenerator<Buffer> {\n  let held: Buffer | undefined;\n  let heldKey: Buffer | undefined;\n  for await (const line of lines) {\n    const key = line.subarray(0, line.indexOf(TAB));\n    if (held !== undefined && heldKey !== undefined && !key.equals(heldKey)) {\n      yield held.subarray(held.indexOf(TAB, heldKey.length + 1) + 1);\n    }\n    held = line;\n    heldKey = key;\n  }\n  if (held !== undefined && heldKey !== undefined)\n    yield held.subarray(held.indexOf(TAB, heldKey.length + 1) + 1);\n}\n\nconst resourceOf = (\n  path: string,\n  facts: Tallied,\n): {\n  readonly name: string;\n  readonly path: string;\n  readonly hash: string;\n  readonly bytes: number;\n} => ({\n  name: basename(path).toLowerCase(),\n  path,\n  hash: `sha256:${facts.sha256}`,\n  bytes: facts.size,\n});\n\n/**\n * This function writes the captures of a store as a WACZ 1.1.1 file. The ZIP holds these parts:\n *\n * - `archive/data.warc.gz`: the WARC file of `exportWarc`, stored with no more compression, so a\n *   replay tool reads a record at its offset.\n * - `indexes/index.cdxj`: the index of the `response` and `revisit` records, sorted by SURT key.\n * - `pages/pages.jsonl`: one page for each document, at its latest capture.\n * - `datapackage.json` and `datapackage-digest.json`: the SHA-256 of each part, and of the package.\n *\n * The index and the page list are sorted on the disk, so the memory does not grow with the store.\n * The export writes temporary files beside `out` and removes them at the end.\n */\nexport const exportWacz = async (\n  source: ArchiveSource,\n  options: ArchiveOptions,\n): Promise<WaczResult> => {\n  const plan = planArchive(source, options);\n  const { target, temporary } = outputOf(options);\n  const parts = `${temporary}.parts`;\n  await mkdir(parts, { recursive: true, mode: 0o700 });\n  try {\n    const created = new Date();\n    const warc = await TalliedFile.create(join(parts, WARC_NAME));\n    const index = new LineSorter(join(parts, \"index\"));\n    const pages = new LineSorter(join(parts, \"pages\"));\n    let counts: ArchiveCounts;\n    try {\n      counts = await writeArchive(plan, warc, WARC_NAME, async (record) => {\n        await index.add(cdxjLine(record, WARC_NAME));\n        if (record.page) {\n          const page = JSON.stringify({ id: record.capture, url: record.url, ts: record.date });\n          await pages.add(`${cacheKeyOf(record.url)}\\t${record.date}\\t${page}`);\n        }\n      });\n    } catch (error) {\n      await warc.abandon();\n      throw error;\n    }\n    const archive = await warc.close();\n\n    const indexFile = await TalliedFile.create(join(parts, \"index.cdxj\"));\n    try {\n      await writeLines(indexFile, index.sorted());\n    } catch (error) {\n      await indexFile.abandon();\n      throw error;\n    }\n    const indexFacts = await indexFile.close();\n\n    const pagesFile = await TalliedFile.create(join(parts, \"pages.jsonl\"));\n    let pageCount: number;\n    try {\n      await pagesFile.write(Buffer.from(`${JSON.stringify(PAGES_HEADER)}\\n`));\n      pageCount = await writeLines(pagesFile, latestOf(pages.sorted()));\n    } catch (error) {\n      await pagesFile.abandon();\n      throw error;\n    }\n    const pagesFacts = await pagesFile.close();\n\n    const datapackage = Buffer.from(\n      JSON.stringify(\n        {\n          profile: \"data-package\",\n          wacz_version: WACZ_VERSION,\n          software: `polite-crawler ${VERSION}`,\n          created: created.toISOString().replace(/\\.\\d{3}Z$/, \"Z\"),\n          title: \"polite-crawler export\",\n          resources: [\n            resourceOf(`archive/${WARC_NAME}`, archive),\n            resourceOf(\"indexes/index.cdxj\", indexFacts),\n            resourceOf(\"pages/pages.jsonl\", pagesFacts),\n          ],\n        },\n        null,\n        2,\n      ),\n      \"utf8\",\n    );\n    const digest = Buffer.from(\n      JSON.stringify(\n        {\n          path: \"datapackage.json\",\n          hash: `sha256:${createHash(\"sha256\").update(datapackage).digest(\"hex\")}`,\n        },\n        null,\n        2,\n      ),\n      \"utf8\",\n    );\n\n    const deflated = async (name: string, facts: Tallied): Promise<ZipMember> => {\n      const file = `${facts.path}.deflate`;\n      return {\n        name,\n        method: 8,\n        crc: facts.crc,\n        size: facts.size,\n        compressedSize: await deflateFile(facts.path, file),\n        file,\n      };\n    };\n    const members: ZipMember[] = [\n      {\n        name: `archive/${WARC_NAME}`,\n        method: 0,\n        crc: archive.crc,\n        size: archive.size,\n        compressedSize: archive.size,\n        file: archive.path,\n      },\n      await deflated(\"indexes/index.cdxj\", indexFacts),\n      await deflated(\"pages/pages.jsonl\", pagesFacts),\n      memberOf(\"datapackage.json\", datapackage),\n      memberOf(\"datapackage-digest.json\", digest),\n    ];\n    let bytes: number;\n    try {\n      bytes = await writeZip(temporary, members, { date: created });\n      await rename(temporary, target);\n    } catch (error) {\n      await rm(temporary, { force: true }).catch(() => undefined);\n      throw error;\n    }\n    return { format: \"wacz\", out: target, bytes, pages: pageCount, ...counts };\n  } finally {\n    await rm(parts, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });\n  }\n};\n"],"mappings":";;;;;;;;;AAIA,MAAMA,YAAU;;AAGhB,MAAM,WAAW,GAAW,MAAsB,OAAO,QAAQ,GAAG,CAAC;;AAGrE,gBAAgB,QAAQ,MAAsC;CAC5D,IAAI;CACJ,WAAW,MAAM,SAAS,iBAAiB,IAAI,GAA4B;EACzE,MAAM,OAAO,SAAS,KAAA,IAAY,QAAQ,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC;EACrE,OAAO,KAAA;EACP,IAAI,QAAQ;EACZ,SAAS;GACP,MAAM,MAAM,KAAK,QAAQA,WAAS,KAAK;GACvC,IAAI,QAAQ,IAAI;GAChB,MAAM,KAAK,SAAS,OAAO,GAAG;GAC9B,QAAQ,MAAM;EAChB;EACA,IAAI,QAAQ,KAAK,QAAQ,OAAO,OAAO,KAAK,KAAK,SAAS,KAAK,CAAC;CAClE;CACA,IAAI,SAAS,KAAA,KAAa,KAAK,SAAS,GAAG,MAAM;AACnD;;;;;;;;;AAUA,IAAa,aAAb,MAAwB;CAMH;CACA;CANnB,QAA0B,CAAC;CAC3B,QAAgB;CAChB,OAAkC,CAAC;CAEnC,YACE,KACA,QAAiC,SACjC;EAFiB,KAAA,MAAA;EACA,KAAA,QAAA;CAChB;;CAGH,IAAI,QAAgB;EAClB,OAAO,KAAK,KAAK;CACnB;CAEA,MAAM,IAAI,MAA6B;EACrC,MAAM,QAAQ,OAAO,KAAK,MAAM,MAAM;EACtC,IAAI,MAAM,SAASA,SAAO,GAAG,MAAM,IAAI,MAAM,mCAAmC;EAChF,KAAK,MAAM,KAAK,KAAK;EAErB,KAAK,SAAS,MAAM,SAAS;EAC7B,IAAI,KAAK,SAAS,KAAK,OAAO,MAAM,KAAK,MAAM;CACjD;CAEA,MAAc,QAAuB;EACnC,IAAI,KAAK,MAAM,WAAW,GAAG;EAC7B,KAAK,MAAM,KAAK,OAAO;EACvB,MAAM,MAAM,KAAK,KAAK,EAAE,WAAW,KAAK,CAAC;EACzC,MAAM,OAAO,KAAK,KAAK,KAAK,GAAG,OAAO,KAAK,KAAK,MAAM,EAAE,OAAO;EAC/D,MAAM,QAAkB,CAAC;EACzB,MAAM,UAAU,OAAO,KAAK,CAACA,SAAO,CAAC;EACrC,KAAK,MAAM,QAAQ,KAAK,OAAO,MAAM,KAAK,MAAM,OAAO;EACvD,MAAM,UAAU,MAAM,OAAO,OAAO,KAAK,GAAG,EAAE,MAAM,IAAM,CAAC;EAC3D,KAAK,KAAK,KAAK,IAAI;EACnB,KAAK,QAAQ,CAAC;EACd,KAAK,QAAQ;CACf;;CAGA,OAAO,SAAiC;EACtC,IAAI,KAAK,KAAK,WAAW,GAAG;GAC1B,MAAM,QAAQ,KAAK,MAAM,KAAK,OAAO;GACrC,KAAK,QAAQ,CAAC;GACd,OAAO;GACP;EACF;EACA,MAAM,KAAK,MAAM;EACjB,MAAM,UAAU,KAAK,KAAK,KAAK,SAAS,QAAQ,IAAI,CAAC,CAAC,OAAO,cAAc,CAAC,CAAC;EAC7E,MAAM,QAAmC,CAAC;EAC1C,KAAK,MAAM,UAAU,SAAS,MAAM,MAAM,MAAM,OAAO,KAAK,EAAA,CAAG,SAAS,KAAA,CAAS;EACjF,SAAS;GACP,IAAI,OAAO;GACX,KAAK,MAAM,CAAC,IAAI,SAAS,MAAM,QAAQ,GACrC,IAAI,SAAS,KAAA,MAAc,SAAS,MAAM,OAAO,QAAQ,MAAM,MAAM,KAAM,IAAI,IAC7E,OAAO;GAEX,IAAI,SAAS,IAAI;GACjB,MAAM,MAAM;GACZ,MAAM,SAAS,MAAM,QAAQ,KAAK,CAAE,KAAK,EAAA,CAAG,SAAS,KAAA;EACvD;CACF;AACF;;;;;AC/EA,MAAM,eAAe;AACrB,MAAM,YAAY;AAClB,MAAM,UAAU,OAAO,KAAK,IAAI;AAChC,MAAM,MAAM;AACZ,MAAM,eAAe;CAAE,QAAQ;CAAkB,IAAI;CAAS,OAAO;AAAY;;AAajF,MAAM,aAAa,OAAO,MAAmB,UAAsD;CACjG,IAAI,QAAsB,CAAC;CAC3B,IAAI,OAAO;CACX,IAAI,QAAQ;CACZ,WAAW,MAAM,QAAQ,OAAO;EAC9B,MAAM,KAAK,MAAM,OAAO;EACxB,QAAQ,KAAK,SAAS;EACtB,SAAS;EACT,IAAI,QAAQ,OAAQ;GAClB,MAAM,KAAK,MAAM,OAAO,OAAO,KAAK,CAAC;GACrC,QAAQ,CAAC;GACT,OAAO;EACT;CACF;CACA,IAAI,MAAM,SAAS,GAAG,MAAM,KAAK,MAAM,OAAO,OAAO,KAAK,CAAC;CAC3D,OAAO;AACT;;;;;;AAOA,gBAAgB,SAAS,OAAsD;CAC7E,IAAI;CACJ,IAAI;CACJ,WAAW,MAAM,QAAQ,OAAO;EAC9B,MAAM,MAAM,KAAK,SAAS,GAAG,KAAK,QAAQ,GAAG,CAAC;EAC9C,IAAI,SAAS,KAAA,KAAa,YAAY,KAAA,KAAa,CAAC,IAAI,OAAO,OAAO,GACpE,MAAM,KAAK,SAAS,KAAK,QAAQ,KAAK,QAAQ,SAAS,CAAC,IAAI,CAAC;EAE/D,OAAO;EACP,UAAU;CACZ;CACA,IAAI,SAAS,KAAA,KAAa,YAAY,KAAA,GACpC,MAAM,KAAK,SAAS,KAAK,QAAQ,KAAK,QAAQ,SAAS,CAAC,IAAI,CAAC;AACjE;AAEA,MAAM,cACJ,MACA,WAMI;CACJ,MAAM,SAAS,IAAI,CAAC,CAAC,YAAY;CACjC;CACA,MAAM,UAAU,MAAM;CACtB,OAAO,MAAM;AACf;;;;;;;;;;;;;AAcA,MAAa,aAAa,OACxB,QACA,YACwB;CACxB,MAAM,OAAO,YAAY,QAAQ,OAAO;CACxC,MAAM,EAAE,QAAQ,cAAc,SAAS,OAAO;CAC9C,MAAM,QAAQ,GAAG,UAAU;CAC3B,MAAM,MAAM,OAAO;EAAE,WAAW;EAAM,MAAM;CAAM,CAAC;CACnD,IAAI;EACF,MAAM,0BAAU,IAAI,KAAK;EACzB,MAAM,OAAO,MAAM,YAAY,OAAO,KAAK,OAAO,SAAS,CAAC;EAC5D,MAAM,QAAQ,IAAI,WAAW,KAAK,OAAO,OAAO,CAAC;EACjD,MAAM,QAAQ,IAAI,WAAW,KAAK,OAAO,OAAO,CAAC;EACjD,IAAI;EACJ,IAAI;GACF,SAAS,MAAM,aAAa,MAAM,MAAM,WAAW,OAAO,WAAW;IACnE,MAAM,MAAM,IAAI,SAAS,QAAQ,SAAS,CAAC;IAC3C,IAAI,OAAO,MAAM;KACf,MAAM,OAAO,KAAK,UAAU;MAAE,IAAI,OAAO;MAAS,KAAK,OAAO;MAAK,IAAI,OAAO;KAAK,CAAC;KACpF,MAAM,MAAM,IAAI,GAAG,WAAW,OAAO,GAAG,EAAE,IAAI,OAAO,KAAK,IAAI,MAAM;IACtE;GACF,CAAC;EACH,SAAS,OAAO;GACd,MAAM,KAAK,QAAQ;GACnB,MAAM;EACR;EACA,MAAM,UAAU,MAAM,KAAK,MAAM;EAEjC,MAAM,YAAY,MAAM,YAAY,OAAO,KAAK,OAAO,YAAY,CAAC;EACpE,IAAI;GACF,MAAM,WAAW,WAAW,MAAM,OAAO,CAAC;EAC5C,SAAS,OAAO;GACd,MAAM,UAAU,QAAQ;GACxB,MAAM;EACR;EACA,MAAM,aAAa,MAAM,UAAU,MAAM;EAEzC,MAAM,YAAY,MAAM,YAAY,OAAO,KAAK,OAAO,aAAa,CAAC;EACrE,IAAI;EACJ,IAAI;GACF,MAAM,UAAU,MAAM,OAAO,KAAK,GAAG,KAAK,UAAU,YAAY,EAAE,GAAG,CAAC;GACtE,YAAY,MAAM,WAAW,WAAW,SAAS,MAAM,OAAO,CAAC,CAAC;EAClE,SAAS,OAAO;GACd,MAAM,UAAU,QAAQ;GACxB,MAAM;EACR;EACA,MAAM,aAAa,MAAM,UAAU,MAAM;EAEzC,MAAM,cAAc,OAAO,KACzB,KAAK,UACH;GACE,SAAS;GACT,cAAc;GACd,UAAU,kBAAkB;GAC5B,SAAS,QAAQ,YAAY,CAAC,CAAC,QAAQ,aAAa,GAAG;GACvD,OAAO;GACP,WAAW;IACT,WAAW,WAAW,aAAa,OAAO;IAC1C,WAAW,sBAAsB,UAAU;IAC3C,WAAW,qBAAqB,UAAU;GAC5C;EACF,GACA,MACA,CACF,GACA,MACF;EACA,MAAM,SAAS,OAAO,KACpB,KAAK,UACH;GACE,MAAM;GACN,MAAM,UAAU,WAAW,QAAQ,CAAC,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,KAAK;EACvE,GACA,MACA,CACF,GACA,MACF;EAEA,MAAM,WAAW,OAAO,MAAc,UAAuC;GAC3E,MAAM,OAAO,GAAG,MAAM,KAAK;GAC3B,OAAO;IACL;IACA,QAAQ;IACR,KAAK,MAAM;IACX,MAAM,MAAM;IACZ,gBAAgB,MAAM,YAAY,MAAM,MAAM,IAAI;IAClD;GACF;EACF;EACA,MAAM,UAAuB;GAC3B;IACE,MAAM,WAAW;IACjB,QAAQ;IACR,KAAK,QAAQ;IACb,MAAM,QAAQ;IACd,gBAAgB,QAAQ;IACxB,MAAM,QAAQ;GAChB;GACA,MAAM,SAAS,sBAAsB,UAAU;GAC/C,MAAM,SAAS,qBAAqB,UAAU;GAC9C,SAAS,oBAAoB,WAAW;GACxC,SAAS,2BAA2B,MAAM;EAC5C;EACA,IAAI;EACJ,IAAI;GACF,QAAQ,MAAM,SAAS,WAAW,SAAS,EAAE,MAAM,QAAQ,CAAC;GAC5D,MAAM,OAAO,WAAW,MAAM;EAChC,SAAS,OAAO;GACd,MAAM,GAAG,WAAW,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;GAC1D,MAAM;EACR;EACA,OAAO;GAAE,QAAQ;GAAQ,KAAK;GAAQ;GAAO,OAAO;GAAW,GAAG;EAAO;CAC3E,UAAU;EACR,MAAM,GAAG,OAAO;GAAE,WAAW;GAAM,OAAO;GAAM,YAAY;GAAG,YAAY;EAAI,CAAC;CAClF;AACF"}