polite-crawler 0.2.4 → 0.3.0

package/build/{version-B_Mpbx8R.mjs → version-Dw3pZOVT.mjs}

@@ -32,7 +32,7 @@ const durationMs = (duration) => {
 //#endregion
 //#region src/version.ts
 /** This is the version of the package. A test makes sure that it agrees with `package.json`. */
-const VERSION = "0.2.4";
+const VERSION = "0.3.0";
 /** This is the product token that robots.txt rules match when an application gives no token. */
 const DEFAULT_TOKEN = "PoliteCrawler";
 /** This page tells a site owner what the crawler is and how to control it. */
@@ -40,4 +40,4 @@ const BOT_PAGE = "https://mark1russell7.github.io/crawler/bot/";
 //#endregion
 export { durationMs as i, DEFAULT_TOKEN as n, VERSION as r, BOT_PAGE as t };
 
-//# sourceMappingURL=version-B_Mpbx8R.mjs.map
+//# sourceMappingURL=version-Dw3pZOVT.mjs.map

package/build/{version-B_Mpbx8R.mjs.map → version-Dw3pZOVT.mjs.map}

@@ -1 +1 @@
-{"version":3,"file":"version-B_Mpbx8R.mjs","names":[],"sources":["../src/duration.ts","../src/version.ts"],"sourcesContent":["/** A duration is a count of milliseconds, or a text, for example `\"30m\"`, `\"24h\"` or `\"7d\"`. */\nexport type Duration = number | string;\n\nconst UNITS: Readonly<Record<string, number>> = {\n  ms: 1,\n  s: 1000,\n  m: 60_000,\n  h: 3_600_000,\n  d: 86_400_000,\n  w: 604_800_000,\n};\n\n/**\n * This function changes a duration into milliseconds. The units are `ms`, `s`, `m` (minutes), `h`,\n * `d` and `w`, in any case. The unit `M` is an error, because a month is not a unit. A bare `0`\n * is zero. The function throws an error for a text that it cannot read.\n *\n * The options take a `Duration` (for example `\"10m\"`), and the crawler reads it. polite-crawler-mcp\n * reads the durations of its tools with it.\n *\n * @beta\n */\nexport const durationMs = (duration: Duration): number => {\n  if (typeof duration === \"number\") {\n    if (!Number.isFinite(duration) || duration < 0)\n      throw new Error(`not a duration: ${String(duration)}`);\n    return duration;\n  }\n  if (/^\\s*0+(\\.0+)?\\s*$/.test(duration)) return 0;\n  const match = /^\\s*(\\d+(?:\\.\\d+)?)\\s*(ms|s|m|h|d|w)\\s*$/i.exec(duration);\n  const unit = match?.[2];\n  const factor = unit === undefined || unit === \"M\" ? undefined : UNITS[unit.toLowerCase()];\n  if (match === null || factor === undefined) {\n    throw new Error(\n      `not a duration: \"${duration}\" (write a number and ms, s, m, h, d or w; m is minutes)`,\n    );\n  }\n  return Number(match[1]) * factor;\n};\n","/** This is the version of the package. A test makes sure that it agrees with `package.json`. */\nexport const VERSION = \"0.2.4\";\n\n/** This is the product token that robots.txt rules match when an application gives no token. */\nexport const DEFAULT_TOKEN = \"PoliteCrawler\";\n\n/** This page tells a site owner what the crawler is and how to control it. */\nexport const BOT_PAGE = \"https://mark1russell7.github.io/crawler/bot/\";\n"],"mappings":";AAGA,MAAM,QAA0C;CAC9C,IAAI;CACJ,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;AACL;;;;;;;;;;;AAYA,MAAa,cAAc,aAA+B;CACxD,IAAI,OAAO,aAAa,UAAU;EAChC,IAAI,CAAC,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC3C,MAAM,IAAI,MAAM,mBAAmB,OAAO,QAAQ,GAAG;EACvD,OAAO;CACT;CACA,IAAI,oBAAoB,KAAK,QAAQ,GAAG,OAAO;CAC/C,MAAM,QAAQ,4CAA4C,KAAK,QAAQ;CACvE,MAAM,OAAO,QAAQ;CACrB,MAAM,SAAS,SAAS,KAAA,KAAa,SAAS,MAAM,KAAA,IAAY,MAAM,KAAK,YAAY;CACvF,IAAI,UAAU,QAAQ,WAAW,KAAA,GAC/B,MAAM,IAAI,MACR,oBAAoB,SAAS,yDAC/B;CAEF,OAAO,OAAO,MAAM,EAAE,IAAI;AAC5B;;;;ACrCA,MAAa,UAAU;;AAGvB,MAAa,gBAAgB;;AAG7B,MAAa,WAAW"}
+{"version":3,"file":"version-Dw3pZOVT.mjs","names":[],"sources":["../src/duration.ts","../src/version.ts"],"sourcesContent":["/** A duration is a count of milliseconds, or a text, for example `\"30m\"`, `\"24h\"` or `\"7d\"`. */\nexport type Duration = number | string;\n\nconst UNITS: Readonly<Record<string, number>> = {\n  ms: 1,\n  s: 1000,\n  m: 60_000,\n  h: 3_600_000,\n  d: 86_400_000,\n  w: 604_800_000,\n};\n\n/**\n * This function changes a duration into milliseconds. The units are `ms`, `s`, `m` (minutes), `h`,\n * `d` and `w`, in any case. The unit `M` is an error, because a month is not a unit. A bare `0`\n * is zero. The function throws an error for a text that it cannot read.\n *\n * The options take a `Duration` (for example `\"10m\"`), and the crawler reads it. polite-crawler-mcp\n * reads the durations of its tools with it.\n *\n * @beta\n */\nexport const durationMs = (duration: Duration): number => {\n  if (typeof duration === \"number\") {\n    if (!Number.isFinite(duration) || duration < 0)\n      throw new Error(`not a duration: ${String(duration)}`);\n    return duration;\n  }\n  if (/^\\s*0+(\\.0+)?\\s*$/.test(duration)) return 0;\n  const match = /^\\s*(\\d+(?:\\.\\d+)?)\\s*(ms|s|m|h|d|w)\\s*$/i.exec(duration);\n  const unit = match?.[2];\n  const factor = unit === undefined || unit === \"M\" ? undefined : UNITS[unit.toLowerCase()];\n  if (match === null || factor === undefined) {\n    throw new Error(\n      `not a duration: \"${duration}\" (write a number and ms, s, m, h, d or w; m is minutes)`,\n    );\n  }\n  return Number(match[1]) * factor;\n};\n","/** This is the version of the package. A test makes sure that it agrees with `package.json`. */\nexport const VERSION = \"0.3.0\";\n\n/** This is the product token that robots.txt rules match when an application gives no token. */\nexport const DEFAULT_TOKEN = \"PoliteCrawler\";\n\n/** This page tells a site owner what the crawler is and how to control it. */\nexport const BOT_PAGE = \"https://mark1russell7.github.io/crawler/bot/\";\n"],"mappings":";AAGA,MAAM,QAA0C;CAC9C,IAAI;CACJ,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;AACL;;;;;;;;;;;AAYA,MAAa,cAAc,aAA+B;CACxD,IAAI,OAAO,aAAa,UAAU;EAChC,IAAI,CAAC,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC3C,MAAM,IAAI,MAAM,mBAAmB,OAAO,QAAQ,GAAG;EACvD,OAAO;CACT;CACA,IAAI,oBAAoB,KAAK,QAAQ,GAAG,OAAO;CAC/C,MAAM,QAAQ,4CAA4C,KAAK,QAAQ;CACvE,MAAM,OAAO,QAAQ;CACrB,MAAM,SAAS,SAAS,KAAA,KAAa,SAAS,MAAM,KAAA,IAAY,MAAM,KAAK,YAAY;CACvF,IAAI,UAAU,QAAQ,WAAW,KAAA,GAC/B,MAAM,IAAI,MACR,oBAAoB,SAAS,yDAC/B;CAEF,OAAO,OAAO,MAAM,EAAE,IAAI;AAC5B;;;;ACrCA,MAAa,UAAU;;AAGvB,MAAa,gBAAgB;;AAG7B,MAAa,WAAW"}

package/build/{wacz-C1EJ8t57.mjs → wacz-ixsUXYCs.mjs}

@@ -1,7 +1,7 @@
 import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.mjs";
-import { r as VERSION } from "./version-B_Mpbx8R.mjs";
+import { r as VERSION } from "./version-Dw3pZOVT.mjs";
 import { t as cacheKeyOf } from "./urls-BHFeSyEK.mjs";
-import { c as deflateFile, i as planArchive, l as memberOf, o as writeArchive, r as outputOf, s as TalliedFile, t as cdxjLine, u as writeZip } from "./warc-CLeYeJNf.mjs";
+import { c as deflateFile, i as planArchive, l as memberOf, o as writeArchive, r as outputOf, s as TalliedFile, t as cdxjLine, u as writeZip } from "./warc-B47Bdjzk.mjs";
 import { createHash } from "node:crypto";
 import { mkdir, rename, rm, writeFile } from "node:fs/promises";
 import { basename, join } from "node:path";
@@ -272,4 +272,4 @@ const exportWacz = async (source, options) => {
 //#endregion
 export { wacz_exports as n, exportWacz as t };
 
-//# sourceMappingURL=wacz-C1EJ8t57.mjs.map
+//# sourceMappingURL=wacz-ixsUXYCs.mjs.map

package/build/{wacz-C1EJ8t57.mjs.map → wacz-ixsUXYCs.mjs.map}

@@ -1 +1 @@
-{"version":3,"file":"wacz-C1EJ8t57.mjs","names":["NEWLINE"],"sources":["../src/sorter.ts","../src/wacz.ts"],"sourcesContent":["import { createReadStream } from \"node:fs\";\nimport { mkdir, writeFile } from \"node:fs/promises\";\nimport { join } from \"node:path\";\n\nconst NEWLINE = 0x0a;\n\n/** This function compares two lines by their bytes. */\nconst byBytes = (a: Buffer, b: Buffer): number => Buffer.compare(a, b);\n\n/** This function reads a file line by line, as bytes. A line holds no line break. */\nasync function* linesOf(path: string): AsyncGenerator<Buffer> {\n  let rest: Buffer | undefined;\n  for await (const chunk of createReadStream(path) as AsyncIterable<Buffer>) {\n    const data = rest === undefined ? chunk : Buffer.concat([rest, chunk]);\n    rest = undefined;\n    let start = 0;\n    for (;;) {\n      const end = data.indexOf(NEWLINE, start);\n      if (end === -1) break;\n      yield data.subarray(start, end);\n      start = end + 1;\n    }\n    if (start < data.length) rest = Buffer.from(data.subarray(start));\n  }\n  if (rest !== undefined && rest.length > 0) yield rest;\n}\n\n/**\n * A line sorter sorts lines of text in the sequence of their UTF-8 bytes (the sequence of\n * `LC_ALL=C sort`). It keeps no more than `limit` bytes of lines in memory. When it has more, it\n * writes them to a sorted file in `dir`. At the end it merges the files. Thus its memory does not\n * grow with the count of lines. The WACZ export sorts its index and its page list with it.\n *\n * A line must not hold a line break.\n */\nexport class LineSorter {\n  private lines: Buffer[] = [];\n  private bytes = 0;\n  private readonly runs: string[] = [];\n\n  constructor(\n    private readonly dir: string,\n    private readonly limit: number = 8 * 1024 * 1024,\n  ) {}\n\n  /** This value is the count of sorted files that the sorter wrote. */\n  get files(): number {\n    return this.runs.length;\n  }\n\n  async add(line: string): Promise<void> {\n    const bytes = Buffer.from(line, \"utf8\");\n    if (bytes.includes(NEWLINE)) throw new Error(\"a line to sort holds a line break\");\n    this.lines.push(bytes);\n    // each line also costs an object in memory, so the count adds a little for each line\n    this.bytes += bytes.length + 64;\n    if (this.bytes >= this.limit) await this.spill();\n  }\n\n  private async spill(): Promise<void> {\n    if (this.lines.length === 0) return;\n    this.lines.sort(byBytes);\n    await mkdir(this.dir, { recursive: true });\n    const path = join(this.dir, `${String(this.runs.length)}.lines`);\n    const parts: Buffer[] = [];\n    const newline = Buffer.from([NEWLINE]);\n    for (const line of this.lines) parts.push(line, newline);\n    await writeFile(path, Buffer.concat(parts), { mode: 0o600 });\n    this.runs.push(path);\n    this.lines = [];\n    this.bytes = 0;\n  }\n\n  /** This method gives each line in the sorted sequence. Use it one time, after the last `add`. */\n  async *sorted(): AsyncGenerator<Buffer> {\n    if (this.runs.length === 0) {\n      const lines = this.lines.sort(byBytes);\n      this.lines = [];\n      yield* lines;\n      return;\n    }\n    await this.spill();\n    const readers = this.runs.map((path) => linesOf(path)[Symbol.asyncIterator]());\n    const heads: Array<Buffer | undefined> = [];\n    for (const reader of readers) heads.push((await reader.next()).value ?? undefined);\n    for (;;) {\n      let pick = -1;\n      for (const [at, head] of heads.entries()) {\n        if (head !== undefined && (pick === -1 || Buffer.compare(head, heads[pick]!) < 0))\n          pick = at;\n      }\n      if (pick === -1) return;\n      yield heads[pick]!;\n      heads[pick] = (await readers[pick]!.next()).value ?? undefined;\n    }\n  }\n}\n","import { createHash } from \"node:crypto\";\nimport { mkdir, rename, rm } from \"node:fs/promises\";\nimport { basename, join } from \"node:path\";\nimport { LineSorter } from \"./sorter.ts\";\nimport { cacheKeyOf } from \"./urls.ts\";\nimport { VERSION } from \"./version.ts\";\nimport type { ArchiveCounts, ArchiveOptions, ArchiveSource } from \"./warc.ts\";\nimport { cdxjLine, outputOf, planArchive, writeArchive } from \"./warc.ts\";\nimport type { Tallied, ZipMember } from \"./zip.ts\";\nimport { TalliedFile, deflateFile, memberOf, writeZip } from \"./zip.ts\";\n\n/*\n * This module writes a WACZ 1.1.1 file: a ZIP with the WARC file, a sorted CDXJ index, a page list\n * and the data package with the SHA-256 of each part. Replay tools (ReplayWeb.page, pywb) open it.\n */\n\n/** This is the version of the WACZ specification that the export follows. */\nconst WACZ_VERSION = \"1.1.1\";\nconst WARC_NAME = \"data.warc.gz\";\nconst NEWLINE = Buffer.from(\"\\n\");\nconst TAB = 0x09;\nconst PAGES_HEADER = { format: \"json-pages-1.0\", id: \"pages\", title: \"All Pages\" };\n\n/** This is the result of `exportWacz`. */\nexport interface WaczResult extends ArchiveCounts {\n  readonly format: \"wacz\";\n  /** This value is the absolute path of the archive. */\n  readonly out: string;\n  readonly bytes: number;\n  /** This value is the count of pages in `pages/pages.jsonl`: one for each document. */\n  readonly pages: number;\n}\n\n/** This function writes lines to a file in batches of approximately 64 KiB. It gives the count of lines. */\nconst writeLines = async (file: TalliedFile, lines: AsyncIterable<Uint8Array>): Promise<number> => {\n  let batch: Uint8Array[] = [];\n  let size = 0;\n  let count = 0;\n  for await (const line of lines) {\n    batch.push(line, NEWLINE);\n    size += line.length + 1;\n    count += 1;\n    if (size >= 65_536) {\n      await file.write(Buffer.concat(batch));\n      batch = [];\n      size = 0;\n    }\n  }\n  if (batch.length > 0) await file.write(Buffer.concat(batch));\n  return count;\n};\n\n/**\n * This function keeps the last line of each document. Each line is the key of the document, a tab,\n * the time, a tab and the JSON of the page. The sorter gives the lines of one key together, the\n * oldest first.\n */\nasync function* latestOf(lines: AsyncIterable<Buffer>): AsyncGenerator<Buffer> {\n  let held: Buffer | undefined;\n  let heldKey: Buffer | undefined;\n  for await (const line of lines) {\n    const key = line.subarray(0, line.indexOf(TAB));\n    if (held !== undefined && heldKey !== undefined && !key.equals(heldKey)) {\n      yield held.subarray(held.indexOf(TAB, heldKey.length + 1) + 1);\n    }\n    held = line;\n    heldKey = key;\n  }\n  if (held !== undefined && heldKey !== undefined)\n    yield held.subarray(held.indexOf(TAB, heldKey.length + 1) + 1);\n}\n\nconst resourceOf = (\n  path: string,\n  facts: Tallied,\n): {\n  readonly name: string;\n  readonly path: string;\n  readonly hash: string;\n  readonly bytes: number;\n} => ({\n  name: basename(path).toLowerCase(),\n  path,\n  hash: `sha256:${facts.sha256}`,\n  bytes: facts.size,\n});\n\n/**\n * This function writes the captures of a store as a WACZ 1.1.1 file. The ZIP holds these parts:\n *\n * - `archive/data.warc.gz`: the WARC file of `exportWarc`, stored with no more compression, so a\n *   replay tool reads a record at its offset.\n * - `indexes/index.cdxj`: the index of the `response` and `revisit` records, sorted by SURT key.\n * - `pages/pages.jsonl`: one page for each document, at its latest capture.\n * - `datapackage.json` and `datapackage-digest.json`: the SHA-256 of each part, and of the package.\n *\n * The index and the page list are sorted on the disk, so the memory does not grow with the store.\n * The export writes temporary files beside `out` and removes them at the end.\n */\nexport const exportWacz = async (\n  source: ArchiveSource,\n  options: ArchiveOptions,\n): Promise<WaczResult> => {\n  const plan = planArchive(source, options);\n  const { target, temporary } = outputOf(options);\n  const parts = `${temporary}.parts`;\n  await mkdir(parts, { recursive: true, mode: 0o700 });\n  try {\n    const created = new Date();\n    const warc = await TalliedFile.create(join(parts, WARC_NAME));\n    const index = new LineSorter(join(parts, \"index\"));\n    const pages = new LineSorter(join(parts, \"pages\"));\n    let counts: ArchiveCounts;\n    try {\n      counts = await writeArchive(plan, warc, WARC_NAME, async (record) => {\n        await index.add(cdxjLine(record, WARC_NAME));\n        if (record.page) {\n          const page = JSON.stringify({ id: record.capture, url: record.url, ts: record.date });\n          await pages.add(`${cacheKeyOf(record.url)}\\t${record.date}\\t${page}`);\n        }\n      });\n    } catch (error) {\n      await warc.abandon();\n      throw error;\n    }\n    const archive = await warc.close();\n\n    const indexFile = await TalliedFile.create(join(parts, \"index.cdxj\"));\n    try {\n      await writeLines(indexFile, index.sorted());\n    } catch (error) {\n      await indexFile.abandon();\n      throw error;\n    }\n    const indexFacts = await indexFile.close();\n\n    const pagesFile = await TalliedFile.create(join(parts, \"pages.jsonl\"));\n    let pageCount: number;\n    try {\n      await pagesFile.write(Buffer.from(`${JSON.stringify(PAGES_HEADER)}\\n`));\n      pageCount = await writeLines(pagesFile, latestOf(pages.sorted()));\n    } catch (error) {\n      await pagesFile.abandon();\n      throw error;\n    }\n    const pagesFacts = await pagesFile.close();\n\n    const datapackage = Buffer.from(\n      JSON.stringify(\n        {\n          profile: \"data-package\",\n          wacz_version: WACZ_VERSION,\n          software: `polite-crawler ${VERSION}`,\n          created: created.toISOString().replace(/\\.\\d{3}Z$/, \"Z\"),\n          title: \"polite-crawler export\",\n          resources: [\n            resourceOf(`archive/${WARC_NAME}`, archive),\n            resourceOf(\"indexes/index.cdxj\", indexFacts),\n            resourceOf(\"pages/pages.jsonl\", pagesFacts),\n          ],\n        },\n        null,\n        2,\n      ),\n      \"utf8\",\n    );\n    const digest = Buffer.from(\n      JSON.stringify(\n        {\n          path: \"datapackage.json\",\n          hash: `sha256:${createHash(\"sha256\").update(datapackage).digest(\"hex\")}`,\n        },\n        null,\n        2,\n      ),\n      \"utf8\",\n    );\n\n    const deflated = async (name: string, facts: Tallied): Promise<ZipMember> => {\n      const file = `${facts.path}.deflate`;\n      return {\n        name,\n        method: 8,\n        crc: facts.crc,\n        size: facts.size,\n        compressedSize: await deflateFile(facts.path, file),\n        file,\n      };\n    };\n    const members: ZipMember[] = [\n      {\n        name: `archive/${WARC_NAME}`,\n        method: 0,\n        crc: archive.crc,\n        size: archive.size,\n        compressedSize: archive.size,\n        file: archive.path,\n      },\n      await deflated(\"indexes/index.cdxj\", indexFacts),\n      await deflated(\"pages/pages.jsonl\", pagesFacts),\n      memberOf(\"datapackage.json\", datapackage),\n      memberOf(\"datapackage-digest.json\", digest),\n    ];\n    let bytes: number;\n    try {\n      bytes = await writeZip(temporary, members, { date: created });\n      await rename(temporary, target);\n    } catch (error) {\n      await rm(temporary, { force: true }).catch(() => undefined);\n      throw error;\n    }\n    return { format: \"wacz\", out: target, bytes, pages: pageCount, ...counts };\n  } finally {\n    await rm(parts, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });\n  }\n};\n"],"mappings":";;;;;;;;;AAIA,MAAMA,YAAU;;AAGhB,MAAM,WAAW,GAAW,MAAsB,OAAO,QAAQ,GAAG,CAAC;;AAGrE,gBAAgB,QAAQ,MAAsC;CAC5D,IAAI;CACJ,WAAW,MAAM,SAAS,iBAAiB,IAAI,GAA4B;EACzE,MAAM,OAAO,SAAS,KAAA,IAAY,QAAQ,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC;EACrE,OAAO,KAAA;EACP,IAAI,QAAQ;EACZ,SAAS;GACP,MAAM,MAAM,KAAK,QAAQA,WAAS,KAAK;GACvC,IAAI,QAAQ,IAAI;GAChB,MAAM,KAAK,SAAS,OAAO,GAAG;GAC9B,QAAQ,MAAM;EAChB;EACA,IAAI,QAAQ,KAAK,QAAQ,OAAO,OAAO,KAAK,KAAK,SAAS,KAAK,CAAC;CAClE;CACA,IAAI,SAAS,KAAA,KAAa,KAAK,SAAS,GAAG,MAAM;AACnD;;;;;;;;;AAUA,IAAa,aAAb,MAAwB;CAMH;CACA;CANnB,QAA0B,CAAC;CAC3B,QAAgB;CAChB,OAAkC,CAAC;CAEnC,YACE,KACA,QAAiC,SACjC;EAFiB,KAAA,MAAA;EACA,KAAA,QAAA;CAChB;;CAGH,IAAI,QAAgB;EAClB,OAAO,KAAK,KAAK;CACnB;CAEA,MAAM,IAAI,MAA6B;EACrC,MAAM,QAAQ,OAAO,KAAK,MAAM,MAAM;EACtC,IAAI,MAAM,SAASA,SAAO,GAAG,MAAM,IAAI,MAAM,mCAAmC;EAChF,KAAK,MAAM,KAAK,KAAK;EAErB,KAAK,SAAS,MAAM,SAAS;EAC7B,IAAI,KAAK,SAAS,KAAK,OAAO,MAAM,KAAK,MAAM;CACjD;CAEA,MAAc,QAAuB;EACnC,IAAI,KAAK,MAAM,WAAW,GAAG;EAC7B,KAAK,MAAM,KAAK,OAAO;EACvB,MAAM,MAAM,KAAK,KAAK,EAAE,WAAW,KAAK,CAAC;EACzC,MAAM,OAAO,KAAK,KAAK,KAAK,GAAG,OAAO,KAAK,KAAK,MAAM,EAAE,OAAO;EAC/D,MAAM,QAAkB,CAAC;EACzB,MAAM,UAAU,OAAO,KAAK,CAACA,SAAO,CAAC;EACrC,KAAK,MAAM,QAAQ,KAAK,OAAO,MAAM,KAAK,MAAM,OAAO;EACvD,MAAM,UAAU,MAAM,OAAO,OAAO,KAAK,GAAG,EAAE,MAAM,IAAM,CAAC;EAC3D,KAAK,KAAK,KAAK,IAAI;EACnB,KAAK,QAAQ,CAAC;EACd,KAAK,QAAQ;CACf;;CAGA,OAAO,SAAiC;EACtC,IAAI,KAAK,KAAK,WAAW,GAAG;GAC1B,MAAM,QAAQ,KAAK,MAAM,KAAK,OAAO;GACrC,KAAK,QAAQ,CAAC;GACd,OAAO;GACP;EACF;EACA,MAAM,KAAK,MAAM;EACjB,MAAM,UAAU,KAAK,KAAK,KAAK,SAAS,QAAQ,IAAI,CAAC,CAAC,OAAO,cAAc,CAAC,CAAC;EAC7E,MAAM,QAAmC,CAAC;EAC1C,KAAK,MAAM,UAAU,SAAS,MAAM,MAAM,MAAM,OAAO,KAAK,EAAA,CAAG,SAAS,KAAA,CAAS;EACjF,SAAS;GACP,IAAI,OAAO;GACX,KAAK,MAAM,CAAC,IAAI,SAAS,MAAM,QAAQ,GACrC,IAAI,SAAS,KAAA,MAAc,SAAS,MAAM,OAAO,QAAQ,MAAM,MAAM,KAAM,IAAI,IAC7E,OAAO;GAEX,IAAI,SAAS,IAAI;GACjB,MAAM,MAAM;GACZ,MAAM,SAAS,MAAM,QAAQ,KAAK,CAAE,KAAK,EAAA,CAAG,SAAS,KAAA;EACvD;CACF;AACF;;;;;AC/EA,MAAM,eAAe;AACrB,MAAM,YAAY;AAClB,MAAM,UAAU,OAAO,KAAK,IAAI;AAChC,MAAM,MAAM;AACZ,MAAM,eAAe;CAAE,QAAQ;CAAkB,IAAI;CAAS,OAAO;AAAY;;AAajF,MAAM,aAAa,OAAO,MAAmB,UAAsD;CACjG,IAAI,QAAsB,CAAC;CAC3B,IAAI,OAAO;CACX,IAAI,QAAQ;CACZ,WAAW,MAAM,QAAQ,OAAO;EAC9B,MAAM,KAAK,MAAM,OAAO;EACxB,QAAQ,KAAK,SAAS;EACtB,SAAS;EACT,IAAI,QAAQ,OAAQ;GAClB,MAAM,KAAK,MAAM,OAAO,OAAO,KAAK,CAAC;GACrC,QAAQ,CAAC;GACT,OAAO;EACT;CACF;CACA,IAAI,MAAM,SAAS,GAAG,MAAM,KAAK,MAAM,OAAO,OAAO,KAAK,CAAC;CAC3D,OAAO;AACT;;;;;;AAOA,gBAAgB,SAAS,OAAsD;CAC7E,IAAI;CACJ,IAAI;CACJ,WAAW,MAAM,QAAQ,OAAO;EAC9B,MAAM,MAAM,KAAK,SAAS,GAAG,KAAK,QAAQ,GAAG,CAAC;EAC9C,IAAI,SAAS,KAAA,KAAa,YAAY,KAAA,KAAa,CAAC,IAAI,OAAO,OAAO,GACpE,MAAM,KAAK,SAAS,KAAK,QAAQ,KAAK,QAAQ,SAAS,CAAC,IAAI,CAAC;EAE/D,OAAO;EACP,UAAU;CACZ;CACA,IAAI,SAAS,KAAA,KAAa,YAAY,KAAA,GACpC,MAAM,KAAK,SAAS,KAAK,QAAQ,KAAK,QAAQ,SAAS,CAAC,IAAI,CAAC;AACjE;AAEA,MAAM,cACJ,MACA,WAMI;CACJ,MAAM,SAAS,IAAI,CAAC,CAAC,YAAY;CACjC;CACA,MAAM,UAAU,MAAM;CACtB,OAAO,MAAM;AACf;;;;;;;;;;;;;AAcA,MAAa,aAAa,OACxB,QACA,YACwB;CACxB,MAAM,OAAO,YAAY,QAAQ,OAAO;CACxC,MAAM,EAAE,QAAQ,cAAc,SAAS,OAAO;CAC9C,MAAM,QAAQ,GAAG,UAAU;CAC3B,MAAM,MAAM,OAAO;EAAE,WAAW;EAAM,MAAM;CAAM,CAAC;CACnD,IAAI;EACF,MAAM,0BAAU,IAAI,KAAK;EACzB,MAAM,OAAO,MAAM,YAAY,OAAO,KAAK,OAAO,SAAS,CAAC;EAC5D,MAAM,QAAQ,IAAI,WAAW,KAAK,OAAO,OAAO,CAAC;EACjD,MAAM,QAAQ,IAAI,WAAW,KAAK,OAAO,OAAO,CAAC;EACjD,IAAI;EACJ,IAAI;GACF,SAAS,MAAM,aAAa,MAAM,MAAM,WAAW,OAAO,WAAW;IACnE,MAAM,MAAM,IAAI,SAAS,QAAQ,SAAS,CAAC;IAC3C,IAAI,OAAO,MAAM;KACf,MAAM,OAAO,KAAK,UAAU;MAAE,IAAI,OAAO;MAAS,KAAK,OAAO;MAAK,IAAI,OAAO;KAAK,CAAC;KACpF,MAAM,MAAM,IAAI,GAAG,WAAW,OAAO,GAAG,EAAE,IAAI,OAAO,KAAK,IAAI,MAAM;IACtE;GACF,CAAC;EACH,SAAS,OAAO;GACd,MAAM,KAAK,QAAQ;GACnB,MAAM;EACR;EACA,MAAM,UAAU,MAAM,KAAK,MAAM;EAEjC,MAAM,YAAY,MAAM,YAAY,OAAO,KAAK,OAAO,YAAY,CAAC;EACpE,IAAI;GACF,MAAM,WAAW,WAAW,MAAM,OAAO,CAAC;EAC5C,SAAS,OAAO;GACd,MAAM,UAAU,QAAQ;GACxB,MAAM;EACR;EACA,MAAM,aAAa,MAAM,UAAU,MAAM;EAEzC,MAAM,YAAY,MAAM,YAAY,OAAO,KAAK,OAAO,aAAa,CAAC;EACrE,IAAI;EACJ,IAAI;GACF,MAAM,UAAU,MAAM,OAAO,KAAK,GAAG,KAAK,UAAU,YAAY,EAAE,GAAG,CAAC;GACtE,YAAY,MAAM,WAAW,WAAW,SAAS,MAAM,OAAO,CAAC,CAAC;EAClE,SAAS,OAAO;GACd,MAAM,UAAU,QAAQ;GACxB,MAAM;EACR;EACA,MAAM,aAAa,MAAM,UAAU,MAAM;EAEzC,MAAM,cAAc,OAAO,KACzB,KAAK,UACH;GACE,SAAS;GACT,cAAc;GACd,UAAU,kBAAkB;GAC5B,SAAS,QAAQ,YAAY,CAAC,CAAC,QAAQ,aAAa,GAAG;GACvD,OAAO;GACP,WAAW;IACT,WAAW,WAAW,aAAa,OAAO;IAC1C,WAAW,sBAAsB,UAAU;IAC3C,WAAW,qBAAqB,UAAU;GAC5C;EACF,GACA,MACA,CACF,GACA,MACF;EACA,MAAM,SAAS,OAAO,KACpB,KAAK,UACH;GACE,MAAM;GACN,MAAM,UAAU,WAAW,QAAQ,CAAC,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,KAAK;EACvE,GACA,MACA,CACF,GACA,MACF;EAEA,MAAM,WAAW,OAAO,MAAc,UAAuC;GAC3E,MAAM,OAAO,GAAG,MAAM,KAAK;GAC3B,OAAO;IACL;IACA,QAAQ;IACR,KAAK,MAAM;IACX,MAAM,MAAM;IACZ,gBAAgB,MAAM,YAAY,MAAM,MAAM,IAAI;IAClD;GACF;EACF;EACA,MAAM,UAAuB;GAC3B;IACE,MAAM,WAAW;IACjB,QAAQ;IACR,KAAK,QAAQ;IACb,MAAM,QAAQ;IACd,gBAAgB,QAAQ;IACxB,MAAM,QAAQ;GAChB;GACA,MAAM,SAAS,sBAAsB,UAAU;GAC/C,MAAM,SAAS,qBAAqB,UAAU;GAC9C,SAAS,oBAAoB,WAAW;GACxC,SAAS,2BAA2B,MAAM;EAC5C;EACA,IAAI;EACJ,IAAI;GACF,QAAQ,MAAM,SAAS,WAAW,SAAS,EAAE,MAAM,QAAQ,CAAC;GAC5D,MAAM,OAAO,WAAW,MAAM;EAChC,SAAS,OAAO;GACd,MAAM,GAAG,WAAW,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;GAC1D,MAAM;EACR;EACA,OAAO;GAAE,QAAQ;GAAQ,KAAK;GAAQ;GAAO,OAAO;GAAW,GAAG;EAAO;CAC3E,UAAU;EACR,MAAM,GAAG,OAAO;GAAE,WAAW;GAAM,OAAO;GAAM,YAAY;GAAG,YAAY;EAAI,CAAC;CAClF;AACF"}
+{"version":3,"file":"wacz-ixsUXYCs.mjs","names":["NEWLINE"],"sources":["../src/sorter.ts","../src/wacz.ts"],"sourcesContent":["import { createReadStream } from \"node:fs\";\nimport { mkdir, writeFile } from \"node:fs/promises\";\nimport { join } from \"node:path\";\n\nconst NEWLINE = 0x0a;\n\n/** This function compares two lines by their bytes. */\nconst byBytes = (a: Buffer, b: Buffer): number => Buffer.compare(a, b);\n\n/** This function reads a file line by line, as bytes. A line holds no line break. */\nasync function* linesOf(path: string): AsyncGenerator<Buffer> {\n  let rest: Buffer | undefined;\n  for await (const chunk of createReadStream(path) as AsyncIterable<Buffer>) {\n    const data = rest === undefined ? chunk : Buffer.concat([rest, chunk]);\n    rest = undefined;\n    let start = 0;\n    for (;;) {\n      const end = data.indexOf(NEWLINE, start);\n      if (end === -1) break;\n      yield data.subarray(start, end);\n      start = end + 1;\n    }\n    if (start < data.length) rest = Buffer.from(data.subarray(start));\n  }\n  if (rest !== undefined && rest.length > 0) yield rest;\n}\n\n/**\n * A line sorter sorts lines of text in the sequence of their UTF-8 bytes (the sequence of\n * `LC_ALL=C sort`). It keeps no more than `limit` bytes of lines in memory. When it has more, it\n * writes them to a sorted file in `dir`. At the end it merges the files. Thus its memory does not\n * grow with the count of lines. The WACZ export sorts its index and its page list with it.\n *\n * A line must not hold a line break.\n */\nexport class LineSorter {\n  private lines: Buffer[] = [];\n  private bytes = 0;\n  private readonly runs: string[] = [];\n\n  constructor(\n    private readonly dir: string,\n    private readonly limit: number = 8 * 1024 * 1024,\n  ) {}\n\n  /** This value is the count of sorted files that the sorter wrote. */\n  get files(): number {\n    return this.runs.length;\n  }\n\n  async add(line: string): Promise<void> {\n    const bytes = Buffer.from(line, \"utf8\");\n    if (bytes.includes(NEWLINE)) throw new Error(\"a line to sort holds a line break\");\n    this.lines.push(bytes);\n    // each line also costs an object in memory, so the count adds a little for each line\n    this.bytes += bytes.length + 64;\n    if (this.bytes >= this.limit) await this.spill();\n  }\n\n  private async spill(): Promise<void> {\n    if (this.lines.length === 0) return;\n    this.lines.sort(byBytes);\n    await mkdir(this.dir, { recursive: true });\n    const path = join(this.dir, `${String(this.runs.length)}.lines`);\n    const parts: Buffer[] = [];\n    const newline = Buffer.from([NEWLINE]);\n    for (const line of this.lines) parts.push(line, newline);\n    await writeFile(path, Buffer.concat(parts), { mode: 0o600 });\n    this.runs.push(path);\n    this.lines = [];\n    this.bytes = 0;\n  }\n\n  /** This method gives each line in the sorted sequence. Use it one time, after the last `add`. */\n  async *sorted(): AsyncGenerator<Buffer> {\n    if (this.runs.length === 0) {\n      const lines = this.lines.sort(byBytes);\n      this.lines = [];\n      yield* lines;\n      return;\n    }\n    await this.spill();\n    const readers = this.runs.map((path) => linesOf(path)[Symbol.asyncIterator]());\n    const heads: Array<Buffer | undefined> = [];\n    for (const reader of readers) heads.push((await reader.next()).value ?? undefined);\n    for (;;) {\n      let pick = -1;\n      for (const [at, head] of heads.entries()) {\n        if (head !== undefined && (pick === -1 || Buffer.compare(head, heads[pick]!) < 0))\n          pick = at;\n      }\n      if (pick === -1) return;\n      yield heads[pick]!;\n      heads[pick] = (await readers[pick]!.next()).value ?? undefined;\n    }\n  }\n}\n","import { createHash } from \"node:crypto\";\nimport { mkdir, rename, rm } from \"node:fs/promises\";\nimport { basename, join } from \"node:path\";\nimport { LineSorter } from \"./sorter.ts\";\nimport { cacheKeyOf } from \"./urls.ts\";\nimport { VERSION } from \"./version.ts\";\nimport type { ArchiveCounts, ArchiveOptions, ArchiveSource } from \"./warc.ts\";\nimport { cdxjLine, outputOf, planArchive, writeArchive } from \"./warc.ts\";\nimport type { Tallied, ZipMember } from \"./zip.ts\";\nimport { TalliedFile, deflateFile, memberOf, writeZip } from \"./zip.ts\";\n\n/*\n * This module writes a WACZ 1.1.1 file: a ZIP with the WARC file, a sorted CDXJ index, a page list\n * and the data package with the SHA-256 of each part. Replay tools (ReplayWeb.page, pywb) open it.\n */\n\n/** This is the version of the WACZ specification that the export follows. */\nconst WACZ_VERSION = \"1.1.1\";\nconst WARC_NAME = \"data.warc.gz\";\nconst NEWLINE = Buffer.from(\"\\n\");\nconst TAB = 0x09;\nconst PAGES_HEADER = { format: \"json-pages-1.0\", id: \"pages\", title: \"All Pages\" };\n\n/** This is the result of `exportWacz`. */\nexport interface WaczResult extends ArchiveCounts {\n  readonly format: \"wacz\";\n  /** This value is the absolute path of the archive. */\n  readonly out: string;\n  readonly bytes: number;\n  /** This value is the count of pages in `pages/pages.jsonl`: one for each document. */\n  readonly pages: number;\n}\n\n/** This function writes lines to a file in batches of approximately 64 KiB. It gives the count of lines. */\nconst writeLines = async (file: TalliedFile, lines: AsyncIterable<Uint8Array>): Promise<number> => {\n  let batch: Uint8Array[] = [];\n  let size = 0;\n  let count = 0;\n  for await (const line of lines) {\n    batch.push(line, NEWLINE);\n    size += line.length + 1;\n    count += 1;\n    if (size >= 65_536) {\n      await file.write(Buffer.concat(batch));\n      batch = [];\n      size = 0;\n    }\n  }\n  if (batch.length > 0) await file.write(Buffer.concat(batch));\n  return count;\n};\n\n/**\n * This function keeps the last line of each document. Each line is the key of the document, a tab,\n * the time, a tab and the JSON of the page. The sorter gives the lines of one key together, the\n * oldest first.\n */\nasync function* latestOf(lines: AsyncIterable<Buffer>): AsyncGenerator<Buffer> {\n  let held: Buffer | undefined;\n  let heldKey: Buffer | undefined;\n  for await (const line of lines) {\n    const key = line.subarray(0, line.indexOf(TAB));\n    if (held !== undefined && heldKey !== undefined && !key.equals(heldKey)) {\n      yield held.subarray(held.indexOf(TAB, heldKey.length + 1) + 1);\n    }\n    held = line;\n    heldKey = key;\n  }\n  if (held !== undefined && heldKey !== undefined)\n    yield held.subarray(held.indexOf(TAB, heldKey.length + 1) + 1);\n}\n\nconst resourceOf = (\n  path: string,\n  facts: Tallied,\n): {\n  readonly name: string;\n  readonly path: string;\n  readonly hash: string;\n  readonly bytes: number;\n} => ({\n  name: basename(path).toLowerCase(),\n  path,\n  hash: `sha256:${facts.sha256}`,\n  bytes: facts.size,\n});\n\n/**\n * This function writes the captures of a store as a WACZ 1.1.1 file. The ZIP holds these parts:\n *\n * - `archive/data.warc.gz`: the WARC file of `exportWarc`, stored with no more compression, so a\n *   replay tool reads a record at its offset.\n * - `indexes/index.cdxj`: the index of the `response` and `revisit` records, sorted by SURT key.\n * - `pages/pages.jsonl`: one page for each document, at its latest capture.\n * - `datapackage.json` and `datapackage-digest.json`: the SHA-256 of each part, and of the package.\n *\n * The index and the page list are sorted on the disk, so the memory does not grow with the store.\n * The export writes temporary files beside `out` and removes them at the end.\n */\nexport const exportWacz = async (\n  source: ArchiveSource,\n  options: ArchiveOptions,\n): Promise<WaczResult> => {\n  const plan = planArchive(source, options);\n  const { target, temporary } = outputOf(options);\n  const parts = `${temporary}.parts`;\n  await mkdir(parts, { recursive: true, mode: 0o700 });\n  try {\n    const created = new Date();\n    const warc = await TalliedFile.create(join(parts, WARC_NAME));\n    const index = new LineSorter(join(parts, \"index\"));\n    const pages = new LineSorter(join(parts, \"pages\"));\n    let counts: ArchiveCounts;\n    try {\n      counts = await writeArchive(plan, warc, WARC_NAME, async (record) => {\n        await index.add(cdxjLine(record, WARC_NAME));\n        if (record.page) {\n          const page = JSON.stringify({ id: record.capture, url: record.url, ts: record.date });\n          await pages.add(`${cacheKeyOf(record.url)}\\t${record.date}\\t${page}`);\n        }\n      });\n    } catch (error) {\n      await warc.abandon();\n      throw error;\n    }\n    const archive = await warc.close();\n\n    const indexFile = await TalliedFile.create(join(parts, \"index.cdxj\"));\n    try {\n      await writeLines(indexFile, index.sorted());\n    } catch (error) {\n      await indexFile.abandon();\n      throw error;\n    }\n    const indexFacts = await indexFile.close();\n\n    const pagesFile = await TalliedFile.create(join(parts, \"pages.jsonl\"));\n    let pageCount: number;\n    try {\n      await pagesFile.write(Buffer.from(`${JSON.stringify(PAGES_HEADER)}\\n`));\n      pageCount = await writeLines(pagesFile, latestOf(pages.sorted()));\n    } catch (error) {\n      await pagesFile.abandon();\n      throw error;\n    }\n    const pagesFacts = await pagesFile.close();\n\n    const datapackage = Buffer.from(\n      JSON.stringify(\n        {\n          profile: \"data-package\",\n          wacz_version: WACZ_VERSION,\n          software: `polite-crawler ${VERSION}`,\n          created: created.toISOString().replace(/\\.\\d{3}Z$/, \"Z\"),\n          title: \"polite-crawler export\",\n          resources: [\n            resourceOf(`archive/${WARC_NAME}`, archive),\n            resourceOf(\"indexes/index.cdxj\", indexFacts),\n            resourceOf(\"pages/pages.jsonl\", pagesFacts),\n          ],\n        },\n        null,\n        2,\n      ),\n      \"utf8\",\n    );\n    const digest = Buffer.from(\n      JSON.stringify(\n        {\n          path: \"datapackage.json\",\n          hash: `sha256:${createHash(\"sha256\").update(datapackage).digest(\"hex\")}`,\n        },\n        null,\n        2,\n      ),\n      \"utf8\",\n    );\n\n    const deflated = async (name: string, facts: Tallied): Promise<ZipMember> => {\n      const file = `${facts.path}.deflate`;\n      return {\n        name,\n        method: 8,\n        crc: facts.crc,\n        size: facts.size,\n        compressedSize: await deflateFile(facts.path, file),\n        file,\n      };\n    };\n    const members: ZipMember[] = [\n      {\n        name: `archive/${WARC_NAME}`,\n        method: 0,\n        crc: archive.crc,\n        size: archive.size,\n        compressedSize: archive.size,\n        file: archive.path,\n      },\n      await deflated(\"indexes/index.cdxj\", indexFacts),\n      await deflated(\"pages/pages.jsonl\", pagesFacts),\n      memberOf(\"datapackage.json\", datapackage),\n      memberOf(\"datapackage-digest.json\", digest),\n    ];\n    let bytes: number;\n    try {\n      bytes = await writeZip(temporary, members, { date: created });\n      await rename(temporary, target);\n    } catch (error) {\n      await rm(temporary, { force: true }).catch(() => undefined);\n      throw error;\n    }\n    return { format: \"wacz\", out: target, bytes, pages: pageCount, ...counts };\n  } finally {\n    await rm(parts, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });\n  }\n};\n"],"mappings":";;;;;;;;;AAIA,MAAMA,YAAU;;AAGhB,MAAM,WAAW,GAAW,MAAsB,OAAO,QAAQ,GAAG,CAAC;;AAGrE,gBAAgB,QAAQ,MAAsC;CAC5D,IAAI;CACJ,WAAW,MAAM,SAAS,iBAAiB,IAAI,GAA4B;EACzE,MAAM,OAAO,SAAS,KAAA,IAAY,QAAQ,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC;EACrE,OAAO,KAAA;EACP,IAAI,QAAQ;EACZ,SAAS;GACP,MAAM,MAAM,KAAK,QAAQA,WAAS,KAAK;GACvC,IAAI,QAAQ,IAAI;GAChB,MAAM,KAAK,SAAS,OAAO,GAAG;GAC9B,QAAQ,MAAM;EAChB;EACA,IAAI,QAAQ,KAAK,QAAQ,OAAO,OAAO,KAAK,KAAK,SAAS,KAAK,CAAC;CAClE;CACA,IAAI,SAAS,KAAA,KAAa,KAAK,SAAS,GAAG,MAAM;AACnD;;;;;;;;;AAUA,IAAa,aAAb,MAAwB;CAMH;CACA;CANnB,QAA0B,CAAC;CAC3B,QAAgB;CAChB,OAAkC,CAAC;CAEnC,YACE,KACA,QAAiC,SACjC;EAFiB,KAAA,MAAA;EACA,KAAA,QAAA;CAChB;;CAGH,IAAI,QAAgB;EAClB,OAAO,KAAK,KAAK;CACnB;CAEA,MAAM,IAAI,MAA6B;EACrC,MAAM,QAAQ,OAAO,KAAK,MAAM,MAAM;EACtC,IAAI,MAAM,SAASA,SAAO,GAAG,MAAM,IAAI,MAAM,mCAAmC;EAChF,KAAK,MAAM,KAAK,KAAK;EAErB,KAAK,SAAS,MAAM,SAAS;EAC7B,IAAI,KAAK,SAAS,KAAK,OAAO,MAAM,KAAK,MAAM;CACjD;CAEA,MAAc,QAAuB;EACnC,IAAI,KAAK,MAAM,WAAW,GAAG;EAC7B,KAAK,MAAM,KAAK,OAAO;EACvB,MAAM,MAAM,KAAK,KAAK,EAAE,WAAW,KAAK,CAAC;EACzC,MAAM,OAAO,KAAK,KAAK,KAAK,GAAG,OAAO,KAAK,KAAK,MAAM,EAAE,OAAO;EAC/D,MAAM,QAAkB,CAAC;EACzB,MAAM,UAAU,OAAO,KAAK,CAACA,SAAO,CAAC;EACrC,KAAK,MAAM,QAAQ,KAAK,OAAO,MAAM,KAAK,MAAM,OAAO;EACvD,MAAM,UAAU,MAAM,OAAO,OAAO,KAAK,GAAG,EAAE,MAAM,IAAM,CAAC;EAC3D,KAAK,KAAK,KAAK,IAAI;EACnB,KAAK,QAAQ,CAAC;EACd,KAAK,QAAQ;CACf;;CAGA,OAAO,SAAiC;EACtC,IAAI,KAAK,KAAK,WAAW,GAAG;GAC1B,MAAM,QAAQ,KAAK,MAAM,KAAK,OAAO;GACrC,KAAK,QAAQ,CAAC;GACd,OAAO;GACP;EACF;EACA,MAAM,KAAK,MAAM;EACjB,MAAM,UAAU,KAAK,KAAK,KAAK,SAAS,QAAQ,IAAI,CAAC,CAAC,OAAO,cAAc,CAAC,CAAC;EAC7E,MAAM,QAAmC,CAAC;EAC1C,KAAK,MAAM,UAAU,SAAS,MAAM,MAAM,MAAM,OAAO,KAAK,EAAA,CAAG,SAAS,KAAA,CAAS;EACjF,SAAS;GACP,IAAI,OAAO;GACX,KAAK,MAAM,CAAC,IAAI,SAAS,MAAM,QAAQ,GACrC,IAAI,SAAS,KAAA,MAAc,SAAS,MAAM,OAAO,QAAQ,MAAM,MAAM,KAAM,IAAI,IAC7E,OAAO;GAEX,IAAI,SAAS,IAAI;GACjB,MAAM,MAAM;GACZ,MAAM,SAAS,MAAM,QAAQ,KAAK,CAAE,KAAK,EAAA,CAAG,SAAS,KAAA;EACvD;CACF;AACF;;;;;AC/EA,MAAM,eAAe;AACrB,MAAM,YAAY;AAClB,MAAM,UAAU,OAAO,KAAK,IAAI;AAChC,MAAM,MAAM;AACZ,MAAM,eAAe;CAAE,QAAQ;CAAkB,IAAI;CAAS,OAAO;AAAY;;AAajF,MAAM,aAAa,OAAO,MAAmB,UAAsD;CACjG,IAAI,QAAsB,CAAC;CAC3B,IAAI,OAAO;CACX,IAAI,QAAQ;CACZ,WAAW,MAAM,QAAQ,OAAO;EAC9B,MAAM,KAAK,MAAM,OAAO;EACxB,QAAQ,KAAK,SAAS;EACtB,SAAS;EACT,IAAI,QAAQ,OAAQ;GAClB,MAAM,KAAK,MAAM,OAAO,OAAO,KAAK,CAAC;GACrC,QAAQ,CAAC;GACT,OAAO;EACT;CACF;CACA,IAAI,MAAM,SAAS,GAAG,MAAM,KAAK,MAAM,OAAO,OAAO,KAAK,CAAC;CAC3D,OAAO;AACT;;;;;;AAOA,gBAAgB,SAAS,OAAsD;CAC7E,IAAI;CACJ,IAAI;CACJ,WAAW,MAAM,QAAQ,OAAO;EAC9B,MAAM,MAAM,KAAK,SAAS,GAAG,KAAK,QAAQ,GAAG,CAAC;EAC9C,IAAI,SAAS,KAAA,KAAa,YAAY,KAAA,KAAa,CAAC,IAAI,OAAO,OAAO,GACpE,MAAM,KAAK,SAAS,KAAK,QAAQ,KAAK,QAAQ,SAAS,CAAC,IAAI,CAAC;EAE/D,OAAO;EACP,UAAU;CACZ;CACA,IAAI,SAAS,KAAA,KAAa,YAAY,KAAA,GACpC,MAAM,KAAK,SAAS,KAAK,QAAQ,KAAK,QAAQ,SAAS,CAAC,IAAI,CAAC;AACjE;AAEA,MAAM,cACJ,MACA,WAMI;CACJ,MAAM,SAAS,IAAI,CAAC,CAAC,YAAY;CACjC;CACA,MAAM,UAAU,MAAM;CACtB,OAAO,MAAM;AACf;;;;;;;;;;;;;AAcA,MAAa,aAAa,OACxB,QACA,YACwB;CACxB,MAAM,OAAO,YAAY,QAAQ,OAAO;CACxC,MAAM,EAAE,QAAQ,cAAc,SAAS,OAAO;CAC9C,MAAM,QAAQ,GAAG,UAAU;CAC3B,MAAM,MAAM,OAAO;EAAE,WAAW;EAAM,MAAM;CAAM,CAAC;CACnD,IAAI;EACF,MAAM,0BAAU,IAAI,KAAK;EACzB,MAAM,OAAO,MAAM,YAAY,OAAO,KAAK,OAAO,SAAS,CAAC;EAC5D,MAAM,QAAQ,IAAI,WAAW,KAAK,OAAO,OAAO,CAAC;EACjD,MAAM,QAAQ,IAAI,WAAW,KAAK,OAAO,OAAO,CAAC;EACjD,IAAI;EACJ,IAAI;GACF,SAAS,MAAM,aAAa,MAAM,MAAM,WAAW,OAAO,WAAW;IACnE,MAAM,MAAM,IAAI,SAAS,QAAQ,SAAS,CAAC;IAC3C,IAAI,OAAO,MAAM;KACf,MAAM,OAAO,KAAK,UAAU;MAAE,IAAI,OAAO;MAAS,KAAK,OAAO;MAAK,IAAI,OAAO;KAAK,CAAC;KACpF,MAAM,MAAM,IAAI,GAAG,WAAW,OAAO,GAAG,EAAE,IAAI,OAAO,KAAK,IAAI,MAAM;IACtE;GACF,CAAC;EACH,SAAS,OAAO;GACd,MAAM,KAAK,QAAQ;GACnB,MAAM;EACR;EACA,MAAM,UAAU,MAAM,KAAK,MAAM;EAEjC,MAAM,YAAY,MAAM,YAAY,OAAO,KAAK,OAAO,YAAY,CAAC;EACpE,IAAI;GACF,MAAM,WAAW,WAAW,MAAM,OAAO,CAAC;EAC5C,SAAS,OAAO;GACd,MAAM,UAAU,QAAQ;GACxB,MAAM;EACR;EACA,MAAM,aAAa,MAAM,UAAU,MAAM;EAEzC,MAAM,YAAY,MAAM,YAAY,OAAO,KAAK,OAAO,aAAa,CAAC;EACrE,IAAI;EACJ,IAAI;GACF,MAAM,UAAU,MAAM,OAAO,KAAK,GAAG,KAAK,UAAU,YAAY,EAAE,GAAG,CAAC;GACtE,YAAY,MAAM,WAAW,WAAW,SAAS,MAAM,OAAO,CAAC,CAAC;EAClE,SAAS,OAAO;GACd,MAAM,UAAU,QAAQ;GACxB,MAAM;EACR;EACA,MAAM,aAAa,MAAM,UAAU,MAAM;EAEzC,MAAM,cAAc,OAAO,KACzB,KAAK,UACH;GACE,SAAS;GACT,cAAc;GACd,UAAU,kBAAkB;GAC5B,SAAS,QAAQ,YAAY,CAAC,CAAC,QAAQ,aAAa,GAAG;GACvD,OAAO;GACP,WAAW;IACT,WAAW,WAAW,aAAa,OAAO;IAC1C,WAAW,sBAAsB,UAAU;IAC3C,WAAW,qBAAqB,UAAU;GAC5C;EACF,GACA,MACA,CACF,GACA,MACF;EACA,MAAM,SAAS,OAAO,KACpB,KAAK,UACH;GACE,MAAM;GACN,MAAM,UAAU,WAAW,QAAQ,CAAC,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,KAAK;EACvE,GACA,MACA,CACF,GACA,MACF;EAEA,MAAM,WAAW,OAAO,MAAc,UAAuC;GAC3E,MAAM,OAAO,GAAG,MAAM,KAAK;GAC3B,OAAO;IACL;IACA,QAAQ;IACR,KAAK,MAAM;IACX,MAAM,MAAM;IACZ,gBAAgB,MAAM,YAAY,MAAM,MAAM,IAAI;IAClD;GACF;EACF;EACA,MAAM,UAAuB;GAC3B;IACE,MAAM,WAAW;IACjB,QAAQ;IACR,KAAK,QAAQ;IACb,MAAM,QAAQ;IACd,gBAAgB,QAAQ;IACxB,MAAM,QAAQ;GAChB;GACA,MAAM,SAAS,sBAAsB,UAAU;GAC/C,MAAM,SAAS,qBAAqB,UAAU;GAC9C,SAAS,oBAAoB,WAAW;GACxC,SAAS,2BAA2B,MAAM;EAC5C;EACA,IAAI;EACJ,IAAI;GACF,QAAQ,MAAM,SAAS,WAAW,SAAS,EAAE,MAAM,QAAQ,CAAC;GAC5D,MAAM,OAAO,WAAW,MAAM;EAChC,SAAS,OAAO;GACd,MAAM,GAAG,WAAW,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;GAC1D,MAAM;EACR;EACA,OAAO;GAAE,QAAQ;GAAQ,KAAK;GAAQ;GAAO,OAAO;GAAW,GAAG;EAAO;CAC3E,UAAU;EACR,MAAM,GAAG,OAAO;GAAE,WAAW;GAAM,OAAO;GAAM,YAAY;GAAG,YAAY;EAAI,CAAC;CAClF;AACF"}

package/build/{warc-CLeYeJNf.mjs → warc-B47Bdjzk.mjs}

@@ -1,6 +1,6 @@
 import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.mjs";
 import { n as FileBlobStore, t as CaptureLog } from "./log-CmBWcyQm.mjs";
-import { r as VERSION, t as BOT_PAGE } from "./version-B_Mpbx8R.mjs";
+import { r as VERSION, t as BOT_PAGE } from "./version-Dw3pZOVT.mjs";
 import { createHash, randomUUID } from "node:crypto";
 import { open, rename, rm, stat } from "node:fs/promises";
 import { basename, join, resolve } from "node:path";
@@ -725,4 +725,4 @@ const exportWarc = async (source, options) => {
 //#endregion
 export { warc_exports as a, deflateFile as c, planArchive as i, memberOf as l, exportWarc as n, writeArchive as o, outputOf as r, TalliedFile as s, cdxjLine as t, writeZip as u };
 
-//# sourceMappingURL=warc-CLeYeJNf.mjs.map
+//# sourceMappingURL=warc-B47Bdjzk.mjs.map

package/build/{warc-CLeYeJNf.mjs.map → warc-B47Bdjzk.mjs.map}

@@ -1 +1 @@
-{"version":3,"file":"warc-CLeYeJNf.mjs","names":[],"sources":["../src/zip.ts","../src/warc.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport { createReadStream, createWriteStream } from \"node:fs\";\nimport type { FileHandle } from \"node:fs/promises\";\nimport { open, stat } from \"node:fs/promises\";\nimport { pipeline } from \"node:stream/promises\";\nimport * as zlib from \"node:zlib\";\n\n/*\n * This module writes a ZIP file with node:zlib only. The WACZ export uses it. It is not part of the\n * API of the package.\n *\n * - A member is stored (method 0) or deflated (method 8).\n * - The writer knows the CRC-32 and the sizes of each member before it writes the member. Thus the\n *   local headers hold the true values, and the file has no data descriptors.\n * - The writer adds the Zip64 records only when a size, an offset or the count of members is too\n *   large for the fields of 32 bits. The option `zip64: \"always\"` adds them in each case, for a test.\n */\n\nconst TABLE: Uint32Array = (() => {\n  const table = new Uint32Array(256);\n  for (let n = 0; n < 256; n += 1) {\n    let c = n;\n    for (let k = 0; k < 8; k += 1) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;\n    table[n] = c >>> 0;\n  }\n  return table;\n})();\n\n/** This function calculates the CRC-32 of ISO 3309 with a table. It is the fallback of `crc32`. */\nexport const tableCrc32 = (bytes: Uint8Array, value = 0): number => {\n  let crc = ~value >>> 0;\n  for (const byte of bytes) crc = TABLE[(crc ^ byte) & 0xff]! ^ (crc >>> 8);\n  return ~crc >>> 0;\n};\n\nconst native = (zlib as { readonly crc32?: (data: Uint8Array, value?: number) => number }).crc32;\n\n/** This function calculates the CRC-32 of the bytes. It continues from `value`. It uses `zlib.crc32` when Node.js has it. */\nexport const crc32 = (bytes: Uint8Array, value = 0): number =>\n  native === undefined ? tableCrc32(bytes, value) : native(bytes, value);\n\n/** These are the facts of a file that a tally gives at its close. */\nexport interface Tallied {\n  readonly path: string;\n  readonly size: number;\n  readonly crc: number;\n  /** This value is the SHA-256 of the bytes, as hexadecimal digits. */\n  readonly sha256: string;\n}\n\n/** This function writes the whole buffer. A write can write less than the buffer, and the loop writes the rest. */\nexport const writeAll = async (handle: FileHandle, bytes: Uint8Array): Promise<void> => {\n  let at = 0;\n  while (at < bytes.length) {\n    const { bytesWritten } = await handle.write(bytes, at, bytes.length - at);\n    at += bytesWritten;\n  }\n};\n\n/** A tally writes a new file in sequence. It counts the bytes and calculates their CRC-32 and SHA-256. */\nexport class TalliedFile {\n  private readonly hash = createHash(\"sha256\");\n  private crcValue = 0;\n  private count = 0;\n\n  private constructor(\n    readonly path: string,\n    private readonly handle: FileHandle,\n  ) {}\n\n  /** This method makes a new file. It fails when the file exists. */\n  static async create(path: string): Promise<TalliedFile> {\n    return new TalliedFile(path, await open(path, \"wx\", 0o600));\n  }\n\n  /** This value is the count of bytes that the tally wrote. It is the offset of the next write. */\n  get size(): number {\n    return this.count;\n  }\n\n  async write(bytes: Uint8Array): Promise<void> {\n    await writeAll(this.handle, bytes);\n    this.hash.update(bytes);\n    this.crcValue = crc32(bytes, this.crcValue);\n    this.count += bytes.length;\n  }\n\n  /** This method closes the file and gives its facts. */\n  async close(): Promise<Tallied> {\n    await this.handle.close();\n    return {\n      path: this.path,\n      size: this.count,\n      crc: this.crcValue,\n      sha256: this.hash.digest(\"hex\"),\n    };\n  }\n\n  /** This method closes the file after an error. It gives no facts. */\n  async abandon(): Promise<void> {\n    await this.handle.close().catch(() => undefined);\n  }\n}\n\n/** This function deflates a file into a new file (raw deflate, as ZIP needs it). It gives the size of the new file. */\nexport const deflateFile = async (from: string, to: string): Promise<number> => {\n  await pipeline(\n    createReadStream(from),\n    zlib.createDeflateRaw(),\n    createWriteStream(to, { flags: \"wx\", mode: 0o600 }),\n  );\n  return (await stat(to)).size;\n};\n\n/** A member of a ZIP file. Its bytes are in a file or in memory, as the ZIP stores them. */\nexport interface ZipMember {\n  /** This value is the path of the member in the ZIP file. */\n  readonly name: string;\n  /** The value 0 stores the bytes. The value 8 holds deflated bytes. */\n  readonly method: 0 | 8;\n  /** This value is the CRC-32 of the bytes before compression. */\n  readonly crc: number;\n  /** This value is the count of bytes before compression. */\n  readonly size: number;\n  /** This value is the count of bytes in the ZIP file. */\n  readonly compressedSize: number;\n  /** This file holds the bytes as the ZIP stores them. */\n  readonly file?: string | undefined;\n  /** These bytes are the member as the ZIP stores them, when `file` is not given. */\n  readonly data?: Uint8Array | undefined;\n}\n\n/** This function makes a member from bytes in memory. A member of method 8 holds the bytes deflated. */\nexport const memberOf = (name: string, bytes: Uint8Array, method: 0 | 8 = 8): ZipMember => {\n  const data = method === 8 ? zlib.deflateRawSync(bytes) : bytes;\n  return {\n    name,\n    method,\n    crc: crc32(bytes),\n    size: bytes.length,\n    compressedSize: data.length,\n    data,\n  };\n};\n\n/** These options change a ZIP file. */\nexport interface ZipOptions {\n  /** This time goes into each member, in the MS-DOS format of ZIP (UTC, to 2 seconds). */\n  readonly date: Date;\n  /** The value `auto` (the default) adds the Zip64 records only when they are necessary. */\n  readonly zip64?: \"auto\" | \"always\" | undefined;\n}\n\nconst MAX32 = 0xffffffff;\nconst MAX16 = 0xffff;\nconst VERSION_ZIP = 20;\nconst VERSION_ZIP64 = 45;\n/** The member was made on Unix (3), with version 4.5 of the specification. */\nconst MADE_BY = (3 << 8) | VERSION_ZIP64;\n/** A plain file that its owner can read and write, and the others can read (`0o100644`). */\nconst FILE_MODE = (0o100644 << 16) >>> 0;\n/** Bit 11 of the flags tells that the name is UTF-8. */\nconst UTF8_NAMES = 0x0800;\n\nconst dosTime = (date: Date): { readonly time: number; readonly day: number } => {\n  const year = Math.min(Math.max(date.getUTCFullYear(), 1980), 2107);\n  return {\n    time: (date.getUTCHours() << 11) | (date.getUTCMinutes() << 5) | (date.getUTCSeconds() >> 1),\n    day: ((year - 1980) << 9) | ((date.getUTCMonth() + 1) << 5) | date.getUTCDate(),\n  };\n};\n\n/** This function makes the Zip64 extra field (id 0x0001) with these values of 64 bits. */\nconst zip64Extra = (values: readonly number[]): Buffer => {\n  const extra = Buffer.alloc(4 + 8 * values.length);\n  extra.writeUInt16LE(0x0001, 0);\n  extra.writeUInt16LE(8 * values.length, 2);\n  values.forEach((value, at) => extra.writeBigUInt64LE(BigInt(value), 4 + 8 * at));\n  return extra;\n};\n\n/**\n * This function writes a ZIP file with these members, in this sequence. It writes a new file and\n * fails when the file exists. It gives the size of the ZIP file.\n */\nexport const writeZip = async (\n  target: string,\n  members: readonly ZipMember[],\n  options: ZipOptions,\n): Promise<number> => {\n  const always = options.zip64 === \"always\";\n  const { time, day } = dosTime(options.date);\n  const handle = await open(target, \"wx\", 0o600);\n  let offset = 0;\n  const put = async (bytes: Uint8Array): Promise<void> => {\n    await writeAll(handle, bytes);\n    offset += bytes.length;\n  };\n  try {\n    const central: Buffer[] = [];\n    for (const member of members) {\n      const name = Buffer.from(member.name, \"utf8\");\n      const start = offset;\n      const bigSizes = always || member.size >= MAX32 || member.compressedSize >= MAX32;\n      const localExtra = bigSizes\n        ? zip64Extra([member.size, member.compressedSize])\n        : Buffer.alloc(0);\n      const local = Buffer.alloc(30);\n      local.writeUInt32LE(0x04034b50, 0);\n      local.writeUInt16LE(bigSizes ? VERSION_ZIP64 : VERSION_ZIP, 4);\n      local.writeUInt16LE(UTF8_NAMES, 6);\n      local.writeUInt16LE(member.method, 8);\n      local.writeUInt16LE(time, 10);\n      local.writeUInt16LE(day, 12);\n      local.writeUInt32LE(member.crc >>> 0, 14);\n      local.writeUInt32LE(bigSizes ? MAX32 : member.compressedSize, 18);\n      local.writeUInt32LE(bigSizes ? MAX32 : member.size, 22);\n      local.writeUInt16LE(name.length, 26);\n      local.writeUInt16LE(localExtra.length, 28);\n      await put(local);\n      await put(name);\n      await put(localExtra);\n      const before = offset;\n      if (member.file !== undefined) {\n        for await (const chunk of createReadStream(member.file) as AsyncIterable<Buffer>)\n          await put(chunk);\n      } else await put(member.data ?? new Uint8Array());\n      if (offset - before !== member.compressedSize)\n        throw new Error(\n          `the member ${member.name} has ${String(offset - before)} bytes, not ${String(member.compressedSize)}`,\n        );\n\n      // the central header names only the values that are too large, in this sequence\n      const bigSize = always || member.size >= MAX32;\n      const bigCompressed = always || member.compressedSize >= MAX32;\n      const bigOffset = always || start >= MAX32;\n      const values = [\n        ...(bigSize ? [member.size] : []),\n        ...(bigCompressed ? [member.compressedSize] : []),\n        ...(bigOffset ? [start] : []),\n      ];\n      const extra = values.length > 0 ? zip64Extra(values) : Buffer.alloc(0);\n      const header = Buffer.alloc(46);\n      header.writeUInt32LE(0x02014b50, 0);\n      header.writeUInt16LE(MADE_BY, 4);\n      header.writeUInt16LE(values.length > 0 ? VERSION_ZIP64 : VERSION_ZIP, 6);\n      header.writeUInt16LE(UTF8_NAMES, 8);\n      header.writeUInt16LE(member.method, 10);\n      header.writeUInt16LE(time, 12);\n      header.writeUInt16LE(day, 14);\n      header.writeUInt32LE(member.crc >>> 0, 16);\n      header.writeUInt32LE(bigCompressed ? MAX32 : member.compressedSize, 20);\n      header.writeUInt32LE(bigSize ? MAX32 : member.size, 24);\n      header.writeUInt16LE(name.length, 28);\n      header.writeUInt16LE(extra.length, 30);\n      header.writeUInt16LE(0, 32);\n      header.writeUInt16LE(0, 34);\n      header.writeUInt16LE(0, 36);\n      header.writeUInt32LE(FILE_MODE, 38);\n      header.writeUInt32LE(bigOffset ? MAX32 : start, 42);\n      central.push(header, name, extra);\n    }\n\n    const centralStart = offset;\n    for (const part of central) await put(part);\n    const centralSize = offset - centralStart;\n    const count = members.length;\n    const zip64End = always || count >= MAX16 || centralStart >= MAX32 || centralSize >= MAX32;\n    if (zip64End) {\n      const recordAt = offset;\n      const record = Buffer.alloc(56);\n      record.writeUInt32LE(0x06064b50, 0);\n      record.writeBigUInt64LE(44n, 4);\n      record.writeUInt16LE(MADE_BY, 12);\n      record.writeUInt16LE(VERSION_ZIP64, 14);\n      record.writeUInt32LE(0, 16);\n      record.writeUInt32LE(0, 20);\n      record.writeBigUInt64LE(BigInt(count), 24);\n      record.writeBigUInt64LE(BigInt(count), 32);\n      record.writeBigUInt64LE(BigInt(centralSize), 40);\n      record.writeBigUInt64LE(BigInt(centralStart), 48);\n      await put(record);\n      const locator = Buffer.alloc(20);\n      locator.writeUInt32LE(0x07064b50, 0);\n      locator.writeUInt32LE(0, 4);\n      locator.writeBigUInt64LE(BigInt(recordAt), 8);\n      locator.writeUInt32LE(1, 16);\n      await put(locator);\n    }\n    const end = Buffer.alloc(22);\n    end.writeUInt32LE(0x06054b50, 0);\n    end.writeUInt16LE(0, 4);\n    end.writeUInt16LE(0, 6);\n    end.writeUInt16LE(zip64End ? MAX16 : count, 8);\n    end.writeUInt16LE(zip64End ? MAX16 : count, 10);\n    end.writeUInt32LE(zip64End ? MAX32 : centralSize, 12);\n    end.writeUInt32LE(zip64End ? MAX32 : centralStart, 16);\n    end.writeUInt16LE(0, 20);\n    await put(end);\n  } finally {\n    await handle.close();\n  }\n  return offset;\n};\n","import { createHash, randomUUID } from \"node:crypto\";\nimport { existsSync } from \"node:fs\";\nimport { rename, rm } from \"node:fs/promises\";\nimport { STATUS_CODES } from \"node:http\";\nimport { basename, join, resolve } from \"node:path\";\nimport { promisify } from \"node:util\";\nimport { gzip } from \"node:zlib\";\nimport type { BlobStore } from \"./blobs.ts\";\nimport { FileBlobStore } from \"./blobs.ts\";\nimport type { CaptureReader } from \"./log.ts\";\nimport { CaptureLog } from \"./log.ts\";\nimport type { CaptureRecord } from \"./record.ts\";\nimport { BOT_PAGE, VERSION } from \"./version.ts\";\nimport { TalliedFile } from \"./zip.ts\";\n\n/*\n * This module writes the capture log as WARC/1.1 (ISO 28500:2017). The README tells what goes into\n * the archive and why. These are the records:\n *\n * - One `warcinfo` record names the software and the filters of the export.\n * - A capture with a body gives a `response` record. Its HTTP header block is rebuilt from the\n *   headers that the capture kept, and its payload is the stored body.\n * - A 304 gives a `revisit` record with the profile `server-not-modified`.\n * - A page that the headless browser rendered gives a `conversion` record: the page after\n *   JavaScript, not the body that the server sent.\n * - Each of them has a `metadata` record (`WARC-Concurrent-To`) with the JSON of the capture.\n *\n * Each record is one gzip member. The export reads the log as a stream and holds one body at a time.\n */\n\nconst gzipAsync = promisify(gzip);\n\nconst HOME = new URL(\"../\", BOT_PAGE).href;\nconst CRLF = \"\\r\\n\";\nconst END = Buffer.from(\"\\r\\n\\r\\n\", \"latin1\");\nconst KINDS: ReadonlySet<string> = new Set([\"capture\", \"revisit\", \"skipped\"]);\nconst HASH = /^[0-9a-f]{64}$/;\n/** The revisit records name their original from this count of recent responses. */\nconst ORIGINALS = 65_536;\nconst SERVER_NOT_MODIFIED = \"http://netpreserve.org/warc/1.1/revisit/server-not-modified\";\nconst HTTP_RESPONSE = \"application/http; msgtype=response\";\n\n/** The source of an export: the folder of a store, or the capture log and the blob store of a runtime. */\nexport type ArchiveSource =\n  string | { readonly captures: CaptureReader; readonly blobs: BlobStore };\n\n/** These filters select the captures of an export. Each filter that you give makes the selection smaller. */\nexport interface ArchiveFilter {\n  /** The export holds the captures received at this time or later. */\n  readonly since?: string | Date | undefined;\n  /** The export holds the captures received before this time. */\n  readonly until?: string | Date | undefined;\n  /** The export holds the captures of this host and of its subdomains, for example `example.com`. */\n  readonly host?: string | undefined;\n  /** The export holds the captures whose URL starts with this text. */\n  readonly urlPrefix?: string | undefined;\n  /** The export holds only the captures with these ids. */\n  readonly ids?: readonly string[] | undefined;\n}\n\n/** These options control an export. Only `out` is necessary. */\nexport interface ArchiveOptions extends ArchiveFilter {\n  /** This file gets the archive. The export writes a temporary file and gives it this name at the end. */\n  readonly out: string;\n  /**\n   * When this value is true, each capture with no body (a refusal, a skip of robots.txt, an error, a\n   * redirect) gets a `metadata` record. The default is false: the export leaves them out.\n   */\n  readonly includeRefusals?: boolean | undefined;\n  /** When this value is true, the export replaces a file that exists. The default is false: the export fails. */\n  readonly overwrite?: boolean | undefined;\n}\n\n/** These counts tell what an export wrote. */\nexport interface ArchiveCounts {\n  /** This value is the count of capture records that the filters selected. */\n  readonly captures: number;\n  readonly responses: number;\n  readonly revisits: number;\n  /** This value is the count of rendered pages (`conversion` records). */\n  readonly conversions: number;\n  readonly metadata: number;\n  /** This value is the count of captures with no body: refusals, skips, errors and redirects. */\n  readonly bodiless: number;\n  /** This value is the count of captures whose body is not in the blob store, or does not agree with its hash. */\n  readonly missing: number;\n  /** This value is the count of damaged lines of the capture log. */\n  readonly damaged: number;\n}\n\n/** This is the result of `exportWarc`. */\nexport interface WarcResult extends ArchiveCounts {\n  readonly format: \"warc\";\n  /** This value is the absolute path of the archive. */\n  readonly out: string;\n  readonly bytes: number;\n}\n\n/** These are the facts of a `response` or `revisit` record that the index of a WACZ file needs. */\nexport interface ArchivedRecord {\n  readonly type: \"response\" | \"revisit\";\n  readonly url: string;\n  readonly date: string;\n  readonly status: number;\n  readonly mime: string;\n  readonly digest: string;\n  readonly offset: number;\n  readonly length: number;\n  readonly capture: string;\n  /** This value is true for a page of the page list: a 200 response that is not a robots.txt file. */\n  readonly page: boolean;\n}\n\n/** A plan is a valid filter and an open source. An export makes it before it writes a file. */\nexport interface ArchivePlan {\n  readonly captures: CaptureReader;\n  readonly blobs: BlobStore;\n  readonly filter: ArchiveSelection;\n  readonly includeRefusals: boolean;\n}\n\n/** This is a filter after the checks: times in ISO 8601, a lowercase host and a set of ids. */\nexport interface ArchiveSelection {\n  readonly since?: string | undefined;\n  readonly until?: string | undefined;\n  readonly host?: string | undefined;\n  readonly urlPrefix?: string | undefined;\n  readonly ids?: ReadonlySet<string> | undefined;\n}\n\nconst RFC4648 = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567\";\n\n/** This function writes bytes in base 32 (RFC 4648), with capitals and padding. WARC tools write SHA-256 digests in this form. */\nexport const base32 = (bytes: Uint8Array): string => {\n  let out = \"\";\n  let value = 0;\n  let bits = 0;\n  for (const byte of bytes) {\n    value = ((value << 8) | byte) & 0xffff;\n    bits += 8;\n    while (bits >= 5) {\n      out += RFC4648[(value >>> (bits - 5)) & 31]!;\n      bits -= 5;\n    }\n  }\n  if (bits > 0) out += RFC4648[(value << (5 - bits)) & 31]!;\n  while (out.length % 8 !== 0) out += \"=\";\n  return out;\n};\n\nconst digestOf = (bytes: Uint8Array): string => `sha256:${base32(bytes)}`;\n\nconst sha256Of = (...parts: readonly Uint8Array[]): Buffer => {\n  const hash = createHash(\"sha256\");\n  for (const part of parts) hash.update(part);\n  return hash.digest();\n};\n\n/** This function gives a version 5 UUID (RFC 9562): the SHA-1 of a namespace and a name. */\nconst uuidV5 = (namespace: string, name: string): string => {\n  const hash = createHash(\"sha1\")\n    .update(Buffer.from(namespace.replace(/-/g, \"\"), \"hex\"))\n    .update(name, \"utf8\")\n    .digest();\n  hash[6] = (hash[6]! & 0x0f) | 0x50;\n  hash[8] = (hash[8]! & 0x3f) | 0x80;\n  const hex = hash.subarray(0, 16).toString(\"hex\");\n  return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;\n};\n\n/** The namespace of the record ids: a UUID of the URL of the site, in the URL namespace of RFC 9562. */\nconst NAMESPACE = uuidV5(\"6ba7b811-9dad-11d1-80b4-00c04fd430c8\", HOME);\n\n/** The id of a record of a capture is the same at each export: it comes from the hash of the capture. */\nconst recordId = (record: CaptureRecord, type: string): string =>\n  `<urn:uuid:${uuidV5(NAMESPACE, `${record.content_hash || `${record.id}@${record.received_at}`}/${type}`)}>`;\n\n/** A field of a header holds no line break. */\nconst clean = (value: string): string => value.replace(/[\\r\\n]+/g, \" \").trim();\n\n/** This function gives one record: the WARC header, the block and the two line breaks at the end. */\nconst recordBytes = (\n  fields: ReadonlyArray<readonly [string, string]>,\n  block: readonly Uint8Array[],\n): Buffer => {\n  const length = block.reduce((sum, part) => sum + part.length, 0);\n  const head = [\n    \"WARC/1.1\",\n    ...fields.map(([name, value]) => `${name}: ${clean(value)}`),\n    `Content-Length: ${String(length)}`,\n    \"\",\n    \"\",\n  ].join(CRLF);\n  return Buffer.concat([Buffer.from(head, \"utf8\"), ...block, END]);\n};\n\n/**\n * This function gives an HTTP status line and header block. HTTP headers are Latin-1 bytes, and\n * undici gives each byte as one character. Thus the block gives back the bytes that the server sent.\n */\nconst httpHead = (status: number, headers: ReadonlyArray<readonly [string, string]>): Buffer =>\n  Buffer.from(\n    [\n      `HTTP/1.1 ${String(status)} ${STATUS_CODES[status] ?? \"Unknown\"}`,\n      ...headers.map(([name, value]) => `${name}: ${clean(value)}`),\n      \"\",\n      \"\",\n    ].join(CRLF),\n    \"latin1\",\n  );\n\n/** These fields of a capture keep a header of the response. */\nconst KEPT_HEADERS: ReadonlyArray<readonly [string, string]> = [\n  [\"etag\", \"ETag\"],\n  [\"last_modified\", \"Last-Modified\"],\n  [\"cache_control\", \"Cache-Control\"],\n  [\"expires\", \"Expires\"],\n  [\"age\", \"Age\"],\n  [\"date\", \"Date\"],\n  [\"vary\", \"Vary\"],\n];\n\n/** These are the facts of a fetch record that the export reads. */\ninterface Facts {\n  readonly url: string;\n  readonly date: string;\n  readonly status: number | undefined;\n  readonly blob: string | undefined;\n  readonly contentType: string | undefined;\n  readonly rendered: boolean;\n  readonly truncated: boolean;\n  readonly cut: boolean;\n  readonly revisitOf: string | undefined;\n  readonly recipe: string | undefined;\n  readonly headers: ReadonlyArray<readonly [string, string]>;\n}\n\nconst text = (value: unknown): string | undefined =>\n  typeof value === \"string\" && value.length > 0 ? value : undefined;\n\n/** This function gives the time of a capture in ISO 8601, with milliseconds. WARC/1.1 permits them. */\nconst isoOf = (value: unknown, fallback: string): string => {\n  const ms = typeof value === \"string\" ? Date.parse(value) : Number.NaN;\n  if (Number.isFinite(ms)) return new Date(ms).toISOString();\n  const other = Date.parse(fallback);\n  return Number.isFinite(other) ? new Date(other).toISOString() : fallback;\n};\n\nconst factsOf = (record: CaptureRecord): Facts | undefined => {\n  const payload = record.payload;\n  if (typeof payload !== \"object\" || payload === null) return undefined;\n  const fields = payload as Readonly<Record<string, unknown>>;\n  const url = text(fields[\"url\"]) ?? text(record.external_id);\n  if (url === undefined) return undefined;\n  const blob = text(fields[\"blob\"]);\n  const status = fields[\"status\"];\n  return {\n    url,\n    date: isoOf(fields[\"fetched_at\"], record.received_at),\n    status: typeof status === \"number\" && Number.isInteger(status) ? status : undefined,\n    blob: blob !== undefined && HASH.test(blob) ? blob : undefined,\n    contentType: text(fields[\"content_type\"]),\n    rendered: fields[\"rendered\"] === true,\n    truncated: fields[\"truncated\"] === true,\n    cut: fields[\"cut\"] === true,\n    revisitOf: text(fields[\"revisit_of\"]),\n    recipe: text(fields[\"recipe\"]),\n    headers: KEPT_HEADERS.flatMap(([field, name]) => {\n      const value = text(fields[field]);\n      return value === undefined ? [] : [[name, value] as const];\n    }),\n  };\n};\n\nconst timeOf = (name: string, value: string | Date | undefined): string | undefined => {\n  if (value === undefined) return undefined;\n  const ms = value instanceof Date ? value.getTime() : Date.parse(value);\n  if (!Number.isFinite(ms)) throw new RangeError(`${name} is not a time: \"${String(value)}\"`);\n  return new Date(ms).toISOString();\n};\n\nconst hostName = (value: string): string | undefined => {\n  try {\n    return new URL(value).hostname.toLowerCase().replace(/\\.$/, \"\");\n  } catch {\n    return undefined;\n  }\n};\n\nconst filterOf = (options: ArchiveFilter): ArchiveSelection => {\n  const since = timeOf(\"since\", options.since);\n  const until = timeOf(\"until\", options.until);\n  let host: string | undefined;\n  if (options.host !== undefined) {\n    host = options.host.trim().toLowerCase().replace(/^\\*\\./, \"\").replace(/\\.$/, \"\");\n    if (host === \"\" || /[/:@\\s]/.test(host))\n      throw new RangeError(`host is a host name, for example example.com, not \"${options.host}\"`);\n  }\n  return {\n    since,\n    until,\n    host,\n    urlPrefix: options.urlPrefix === \"\" ? undefined : options.urlPrefix,\n    ids: options.ids === undefined ? undefined : new Set(options.ids),\n  };\n};\n\nconst selected = (record: CaptureRecord, facts: Facts, filter: ArchiveSelection): boolean => {\n  if (filter.since !== undefined && record.received_at < filter.since) return false;\n  if (filter.until !== undefined && record.received_at >= filter.until) return false;\n  if (filter.urlPrefix !== undefined && !facts.url.startsWith(filter.urlPrefix)) return false;\n  if (filter.ids !== undefined && !filter.ids.has(record.id)) return false;\n  if (filter.host !== undefined) {\n    const host = hostName(facts.url);\n    if (host === undefined || (host !== filter.host && !host.endsWith(`.${filter.host}`)))\n      return false;\n  }\n  return true;\n};\n\n/**\n * This function checks the filters and opens the source. It writes nothing. A folder must hold a\n * capture log, and the export does not make a store.\n */\nexport const planArchive = (source: ArchiveSource, options: ArchiveOptions): ArchivePlan => {\n  const filter = filterOf(options);\n  const includeRefusals = options.includeRefusals === true;\n  if (typeof source !== \"string\") {\n    return { captures: source.captures, blobs: source.blobs, filter, includeRefusals };\n  }\n  const dir = resolve(source);\n  if (!existsSync(join(dir, \"captures\")))\n    throw new Error(`the store ${dir} has no capture log (captures/)`);\n  return {\n    captures: new CaptureLog(join(dir, \"captures\"), { sync: false }),\n    blobs: new FileBlobStore(join(dir, \"blobs\"), { sync: false }),\n    filter,\n    includeRefusals,\n  };\n};\n\n/** This function gives the path of the archive and of its temporary file. It fails when the archive exists, unless `overwrite` is true. */\nexport const outputOf = (\n  options: ArchiveOptions,\n): { readonly target: string; readonly temporary: string } => {\n  if (typeof options.out !== \"string\" || options.out.trim() === \"\")\n    throw new Error(\"an export needs out: the file of the archive\");\n  const target = resolve(options.out);\n  if (options.overwrite !== true && existsSync(target))\n    throw new Error(`${target} exists: give a different file, or set overwrite`);\n  return { target, temporary: `${target}.${randomUUID()}.tmp` };\n};\n\n/** This function reads a body from the blob store. It gives no body when the bytes do not agree with the hash. */\nconst bodyOf = async (\n  blobs: BlobStore,\n  hash: string,\n): Promise<{ readonly bytes: Uint8Array; readonly digest: Buffer } | undefined> => {\n  let bytes: Uint8Array | undefined;\n  try {\n    bytes = await blobs.get(hash);\n  } catch {\n    return undefined;\n  }\n  if (bytes === undefined) return undefined;\n  const digest = sha256Of(bytes);\n  return digest.toString(\"hex\") === hash ? { bytes, digest } : undefined;\n};\n\nconst mimeOf = (contentType: string | undefined): string =>\n  contentType?.split(\";\")[0]?.trim().toLowerCase() || \"-\";\n\n/** This function gives the JSON of the metadata record of a capture. The labels and the scope of the application stay out. */\nconst metadataOf = (record: CaptureRecord, written: Readonly<Record<string, string>>): Buffer =>\n  Buffer.from(\n    `${JSON.stringify({\n      capture: {\n        id: record.id,\n        source: record.source,\n        kind: record.kind,\n        ...(record.external_id !== undefined ? { external_id: record.external_id } : {}),\n        ...(record.occurred_at !== undefined ? { occurred_at: record.occurred_at } : {}),\n        received_at: record.received_at,\n        content_hash: record.content_hash,\n        payload: record.payload,\n      },\n      export: written,\n    })}\\n`,\n    \"utf8\",\n  );\n\n/**\n * This function writes the records of an archive into a file: the `warcinfo` record, then the\n * records of each capture that the plan selects. It gives the facts of each `response` and\n * `revisit` record to `onRecord`, for the index of a WACZ file.\n */\nexport const writeArchive = async (\n  plan: ArchivePlan,\n  file: TalliedFile,\n  filename: string,\n  onRecord?: (record: ArchivedRecord) => Promise<void>,\n): Promise<ArchiveCounts> => {\n  const counts = {\n    captures: 0,\n    responses: 0,\n    revisits: 0,\n    conversions: 0,\n    metadata: 0,\n    bodiless: 0,\n    missing: 0,\n    damaged: 0,\n  };\n  const { filter } = plan;\n  const put = async (\n    bytes: Buffer,\n  ): Promise<{ readonly offset: number; readonly length: number }> => {\n    const member = await gzipAsync(bytes);\n    const offset = file.size;\n    await file.write(member);\n    return { offset, length: member.length };\n  };\n\n  const info = `<urn:uuid:${randomUUID()}>`;\n  const shown = Object.fromEntries(\n    Object.entries({\n      since: filter.since,\n      until: filter.until,\n      host: filter.host,\n      url_prefix: filter.urlPrefix,\n      ids: filter.ids === undefined ? undefined : filter.ids.size,\n      include_refusals: plan.includeRefusals,\n    }).filter(([, value]) => value !== undefined),\n  );\n  const fields = [\n    `software: polite-crawler/${VERSION} (+${HOME})`,\n    \"format: WARC File Format 1.1\",\n    \"conformsTo: https://iipc.github.io/warc-specifications/specifications/warc-format/warc-1.1/\",\n    \"robots: obey\",\n    \"description: An export of the capture log of polite-crawler. The HTTP headers of each response are rebuilt from the headers that the capture kept.\",\n    `polite-crawler-filter: ${JSON.stringify(shown)}`,\n  ];\n  await put(\n    recordBytes(\n      [\n        [\"WARC-Type\", \"warcinfo\"],\n        [\"WARC-Record-ID\", info],\n        [\"WARC-Date\", new Date().toISOString()],\n        [\"WARC-Filename\", filename],\n        [\"Content-Type\", \"application/warc-fields\"],\n      ],\n      [Buffer.from(`${fields.join(CRLF)}${CRLF}`, \"utf8\")],\n    ),\n  );\n\n  // a revisit names its original when the original is one of the recent responses of the export\n  const originals = new Map<\n    string,\n    { readonly id: string; readonly date: string; readonly url: string }\n  >();\n  const remember = (\n    capture: string,\n    original: { readonly id: string; readonly date: string; readonly url: string },\n  ): void => {\n    originals.delete(capture);\n    originals.set(capture, original);\n    if (originals.size > ORIGINALS) originals.delete(originals.keys().next().value!);\n  };\n\n  const scan = plan.captures.scan(\n    { source: \"fetch\", since: filter.since },\n    { onDamaged: () => void (counts.damaged += 1) },\n  );\n  for await (const record of scan) {\n    if (record.source !== \"fetch\" || !KINDS.has(record.kind)) continue;\n    const facts = factsOf(record);\n    if (facts === undefined || !selected(record, facts, filter)) continue;\n    counts.captures += 1;\n    const common: Array<readonly [string, string]> = [\n      [\"WARC-Date\", facts.date],\n      [\"WARC-Target-URI\", facts.url],\n      [\"WARC-Warcinfo-ID\", info],\n    ];\n    const vendor: Array<readonly [string, string]> = [[\"WARC-Polite-Crawler-Capture\", record.id]];\n    let main: string | undefined;\n    let written: Record<string, string> = { record: \"none\", reason: \"no body\" };\n\n    if (record.kind === \"revisit\" && facts.blob !== undefined) {\n      // the 304 confirmed the stored body, so the payload digest is the digest of that body\n      const original = facts.revisitOf === undefined ? undefined : originals.get(facts.revisitOf);\n      const head = httpHead(304, [\n        ...(facts.contentType !== undefined ? [[\"Content-Type\", facts.contentType] as const] : []),\n        ...facts.headers,\n      ]);\n      const digest = digestOf(Buffer.from(facts.blob, \"hex\"));\n      main = recordId(record, \"revisit\");\n      const at = await put(\n        recordBytes(\n          [\n            [\"WARC-Type\", \"revisit\"],\n            [\"WARC-Record-ID\", main],\n            ...common,\n            [\"WARC-Profile\", SERVER_NOT_MODIFIED],\n            ...(original !== undefined ? [[\"WARC-Refers-To\", original.id] as const] : []),\n            [\"WARC-Refers-To-Target-URI\", original?.url ?? facts.url],\n            ...(original !== undefined ? [[\"WARC-Refers-To-Date\", original.date] as const] : []),\n            [\"WARC-Payload-Digest\", digest],\n            [\"WARC-Block-Digest\", digestOf(sha256Of(head))],\n            ...vendor,\n            [\"WARC-Polite-Crawler-Headers\", \"rebuilt\"],\n            [\"Content-Type\", HTTP_RESPONSE],\n          ],\n          [head],\n        ),\n      );\n      counts.revisits += 1;\n      written = { record: \"revisit\", http_headers: \"rebuilt\" };\n      await onRecord?.({\n        type: \"revisit\",\n        url: facts.url,\n        date: facts.date,\n        status: 304,\n        mime: \"warc/revisit\",\n        digest,\n        ...at,\n        capture: record.id,\n        page: false,\n      });\n    } else if (record.kind === \"capture\" && facts.blob !== undefined) {\n      const body = await bodyOf(plan.blobs, facts.blob);\n      if (body === undefined) {\n        counts.missing += 1;\n        written = { record: \"none\", reason: \"body missing\" };\n      } else if (facts.rendered) {\n        // the page after JavaScript is a conversion of the page, not the body that the server sent\n        const digest = digestOf(body.digest);\n        main = recordId(record, \"conversion\");\n        await put(\n          recordBytes(\n            [\n              [\"WARC-Type\", \"conversion\"],\n              [\"WARC-Record-ID\", main],\n              ...common,\n              [\"WARC-Payload-Digest\", digest],\n              [\"WARC-Block-Digest\", digest],\n              ...vendor,\n              [\"Content-Type\", facts.contentType ?? \"text/html; charset=utf-8\"],\n            ],\n            [body.bytes],\n          ),\n        );\n        counts.conversions += 1;\n        written = { record: \"conversion\", body: \"rendered\" };\n      } else if (facts.status !== undefined && facts.status >= 100 && facts.status <= 999) {\n        const head = httpHead(facts.status, [\n          ...(facts.contentType !== undefined\n            ? [[\"Content-Type\", facts.contentType] as const]\n            : []),\n          [\"Content-Length\", String(body.bytes.length)],\n          ...facts.headers,\n        ]);\n        const digest = digestOf(body.digest);\n        main = recordId(record, \"response\");\n        // a body that the crawler cut at its limit, or that ended early, says so\n        const truncated = facts.cut ? \"disconnect\" : facts.truncated ? \"length\" : undefined;\n        const at = await put(\n          recordBytes(\n            [\n              [\"WARC-Type\", \"response\"],\n              [\"WARC-Record-ID\", main],\n              ...common,\n              [\"WARC-Payload-Digest\", digest],\n              [\"WARC-Block-Digest\", digestOf(sha256Of(head, body.bytes))],\n              ...(truncated !== undefined ? [[\"WARC-Truncated\", truncated] as const] : []),\n              ...vendor,\n              [\"WARC-Polite-Crawler-Headers\", \"rebuilt\"],\n              [\"Content-Type\", HTTP_RESPONSE],\n            ],\n            [head, body.bytes],\n          ),\n        );\n        counts.responses += 1;\n        written = { record: \"response\", http_headers: \"rebuilt\", body: \"decoded\" };\n        remember(record.id, { id: main, date: facts.date, url: facts.url });\n        await onRecord?.({\n          type: \"response\",\n          url: facts.url,\n          date: facts.date,\n          status: facts.status,\n          mime: mimeOf(facts.contentType),\n          digest,\n          ...at,\n          capture: record.id,\n          page: facts.status === 200 && facts.recipe !== \"robots.txt\",\n        });\n      }\n    }\n\n    if (main === undefined) {\n      if (written[\"reason\"] === \"no body\") counts.bodiless += 1;\n      if (!plan.includeRefusals) continue;\n    }\n    const block = metadataOf(record, written);\n    await put(\n      recordBytes(\n        [\n          [\"WARC-Type\", \"metadata\"],\n          [\"WARC-Record-ID\", recordId(record, \"metadata\")],\n          ...common,\n          ...(main !== undefined ? [[\"WARC-Concurrent-To\", main] as const] : []),\n          [\"WARC-Block-Digest\", digestOf(sha256Of(block))],\n          ...vendor,\n          [\"Content-Type\", \"application/json\"],\n        ],\n        [block],\n      ),\n    );\n    counts.metadata += 1;\n  }\n  return counts;\n};\n\n/**\n * This function gives the SURT key of a URL, as `pywb` and the `surt` library make it with their\n * defaults. The host is reversed with commas and has no `www.` and no default port. The path\n * and the query are lowercase, the path has no slash at the end, and the parts of the query are\n * sorted.\n */\nexport const surt = (value: string): string => {\n  let url: URL;\n  try {\n    url = new URL(value);\n  } catch {\n    return value.toLowerCase();\n  }\n  const host = url.hostname\n    .toLowerCase()\n    .replace(/\\.$/, \"\")\n    .replace(/^www\\d*\\./, \"\");\n  const key = host.startsWith(\"[\") ? host : host.split(\".\").reverse().join(\",\");\n  let path = url.pathname.toLowerCase();\n  if (path.length > 1 && path.endsWith(\"/\")) path = path.slice(0, -1);\n  const query = url.search.slice(1).toLowerCase().split(\"&\").filter(Boolean).sort().join(\"&\");\n  return `${key}${url.port === \"\" ? \"\" : `:${url.port}`})${path}${query === \"\" ? \"\" : `?${query}`}`;\n};\n\n/** This function gives the line of a record in a CDXJ index: the SURT key, the time to the second, and JSON. */\nexport const cdxjLine = (record: ArchivedRecord, filename: string): string =>\n  `${surt(record.url)} ${record.date.slice(0, 19).replace(/\\D/g, \"\")} ${JSON.stringify({\n    url: record.url,\n    mime: record.mime,\n    status: String(record.status),\n    digest: record.digest,\n    length: String(record.length),\n    offset: String(record.offset),\n    filename,\n  })}`;\n\n/**\n * This function writes the captures of a store as a WARC/1.1 file, with one gzip member for each\n * record (`.warc.gz`). It reads the capture log as a stream, so its memory does not grow with the\n * store. It writes a temporary file and gives it the name `out` at the end, so a failed export\n * leaves no part of a file.\n *\n * The source is the folder of a store, or the capture log and the blob store of a runtime\n * (`crawler.runtime`). The export only reads the store.\n */\nexport const exportWarc = async (\n  source: ArchiveSource,\n  options: ArchiveOptions,\n): Promise<WarcResult> => {\n  const plan = planArchive(source, options);\n  const { target, temporary } = outputOf(options);\n  const file = await TalliedFile.create(temporary);\n  try {\n    const counts = await writeArchive(plan, file, basename(target));\n    const done = await file.close();\n    await rename(temporary, target);\n    return { format: \"warc\", out: target, bytes: done.size, ...counts };\n  } catch (error) {\n    await file.abandon();\n    await rm(temporary, { force: true }).catch(() => undefined);\n    throw error;\n  }\n};\n"],"mappings":";;;;;;;;;;;;;AAkBA,MAAM,eAA4B;CAChC,MAAM,wBAAQ,IAAI,YAAY,GAAG;CACjC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK,GAAG;EAC/B,IAAI,IAAI;EACR,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,IAAI,IAAI,aAAc,MAAM,IAAK,MAAM;EAC1E,MAAM,KAAK,MAAM;CACnB;CACA,OAAO;AACT,EAAA,CAAG;;AAGH,MAAa,cAAc,OAAmB,QAAQ,MAAc;CAClE,IAAI,MAAM,CAAC,UAAU;CACrB,KAAK,MAAM,QAAQ,OAAO,MAAM,OAAO,MAAM,QAAQ,OAAU,QAAQ;CACvE,OAAO,CAAC,QAAQ;AAClB;AAEA,MAAM,SAAU,KAA2E;;AAG3F,MAAa,SAAS,OAAmB,QAAQ,MAC/C,WAAW,KAAA,IAAY,WAAW,OAAO,KAAK,IAAI,OAAO,OAAO,KAAK;;AAYvE,MAAa,WAAW,OAAO,QAAoB,UAAqC;CACtF,IAAI,KAAK;CACT,OAAO,KAAK,MAAM,QAAQ;EACxB,MAAM,EAAE,iBAAiB,MAAM,OAAO,MAAM,OAAO,IAAI,MAAM,SAAS,EAAE;EACxE,MAAM;CACR;AACF;;AAGA,IAAa,cAAb,MAAa,YAAY;CAMZ;CACQ;CANnB,OAAwB,WAAW,QAAQ;CAC3C,WAAmB;CACnB,QAAgB;CAEhB,YACE,MACA,QACA;EAFS,KAAA,OAAA;EACQ,KAAA,SAAA;CAChB;;CAGH,aAAa,OAAO,MAAoC;EACtD,OAAO,IAAI,YAAY,MAAM,MAAM,KAAK,MAAM,MAAM,GAAK,CAAC;CAC5D;;CAGA,IAAI,OAAe;EACjB,OAAO,KAAK;CACd;CAEA,MAAM,MAAM,OAAkC;EAC5C,MAAM,SAAS,KAAK,QAAQ,KAAK;EACjC,KAAK,KAAK,OAAO,KAAK;EACtB,KAAK,WAAW,MAAM,OAAO,KAAK,QAAQ;EAC1C,KAAK,SAAS,MAAM;CACtB;;CAGA,MAAM,QAA0B;EAC9B,MAAM,KAAK,OAAO,MAAM;EACxB,OAAO;GACL,MAAM,KAAK;GACX,MAAM,KAAK;GACX,KAAK,KAAK;GACV,QAAQ,KAAK,KAAK,OAAO,KAAK;EAChC;CACF;;CAGA,MAAM,UAAyB;EAC7B,MAAM,KAAK,OAAO,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;CACjD;AACF;;AAGA,MAAa,cAAc,OAAO,MAAc,OAAgC;CAC9E,MAAM,SACJ,iBAAiB,IAAI,GACrB,KAAK,iBAAiB,GACtB,kBAAkB,IAAI;EAAE,OAAO;EAAM,MAAM;CAAM,CAAC,CACpD;CACA,QAAQ,MAAM,KAAK,EAAE,EAAA,CAAG;AAC1B;;AAqBA,MAAa,YAAY,MAAc,OAAmB,SAAgB,MAAiB;CACzF,MAAM,OAAO,WAAW,IAAI,KAAK,eAAe,KAAK,IAAI;CACzD,OAAO;EACL;EACA;EACA,KAAK,MAAM,KAAK;EAChB,MAAM,MAAM;EACZ,gBAAgB,KAAK;EACrB;CACF;AACF;AAUA,MAAM,QAAQ;AACd,MAAM,QAAQ;AACd,MAAM,cAAc;AACpB,MAAM,gBAAgB;;AAEtB,MAAM,UAAU;;AAEhB,MAAM,YAAa,SAAY,OAAQ;;AAEvC,MAAM,aAAa;AAEnB,MAAM,WAAW,SAAgE;CAC/E,MAAM,OAAO,KAAK,IAAI,KAAK,IAAI,KAAK,eAAe,GAAG,IAAI,GAAG,IAAI;CACjE,OAAO;EACL,MAAO,KAAK,YAAY,KAAK,KAAO,KAAK,cAAc,KAAK,IAAM,KAAK,cAAc,KAAK;EAC1F,KAAO,OAAO,QAAS,IAAO,KAAK,YAAY,IAAI,KAAM,IAAK,KAAK,WAAW;CAChF;AACF;;AAGA,MAAM,cAAc,WAAsC;CACxD,MAAM,QAAQ,OAAO,MAAM,IAAI,IAAI,OAAO,MAAM;CAChD,MAAM,cAAc,GAAQ,CAAC;CAC7B,MAAM,cAAc,IAAI,OAAO,QAAQ,CAAC;CACxC,OAAO,SAAS,OAAO,OAAO,MAAM,iBAAiB,OAAO,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;CAC/E,OAAO;AACT;;;;;AAMA,MAAa,WAAW,OACtB,QACA,SACA,YACoB;CACpB,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,EAAE,MAAM,QAAQ,QAAQ,QAAQ,IAAI;CAC1C,MAAM,SAAS,MAAM,KAAK,QAAQ,MAAM,GAAK;CAC7C,IAAI,SAAS;CACb,MAAM,MAAM,OAAO,UAAqC;EACtD,MAAM,SAAS,QAAQ,KAAK;EAC5B,UAAU,MAAM;CAClB;CACA,IAAI;EACF,MAAM,UAAoB,CAAC;EAC3B,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,OAAO,OAAO,KAAK,OAAO,MAAM,MAAM;GAC5C,MAAM,QAAQ;GACd,MAAM,WAAW,UAAU,OAAO,QAAQ,SAAS,OAAO,kBAAkB;GAC5E,MAAM,aAAa,WACf,WAAW,CAAC,OAAO,MAAM,OAAO,cAAc,CAAC,IAC/C,OAAO,MAAM,CAAC;GAClB,MAAM,QAAQ,OAAO,MAAM,EAAE;GAC7B,MAAM,cAAc,UAAY,CAAC;GACjC,MAAM,cAAc,WAAW,gBAAgB,aAAa,CAAC;GAC7D,MAAM,cAAc,YAAY,CAAC;GACjC,MAAM,cAAc,OAAO,QAAQ,CAAC;GACpC,MAAM,cAAc,MAAM,EAAE;GAC5B,MAAM,cAAc,KAAK,EAAE;GAC3B,MAAM,cAAc,OAAO,QAAQ,GAAG,EAAE;GACxC,MAAM,cAAc,WAAW,QAAQ,OAAO,gBAAgB,EAAE;GAChE,MAAM,cAAc,WAAW,QAAQ,OAAO,MAAM,EAAE;GACtD,MAAM,cAAc,KAAK,QAAQ,EAAE;GACnC,MAAM,cAAc,WAAW,QAAQ,EAAE;GACzC,MAAM,IAAI,KAAK;GACf,MAAM,IAAI,IAAI;GACd,MAAM,IAAI,UAAU;GACpB,MAAM,SAAS;GACf,IAAI,OAAO,SAAS,KAAA,GAClB,WAAW,MAAM,SAAS,iBAAiB,OAAO,IAAI,GACpD,MAAM,IAAI,KAAK;QACZ,MAAM,IAAI,OAAO,wBAAQ,IAAI,WAAW,CAAC;GAChD,IAAI,SAAS,WAAW,OAAO,gBAC7B,MAAM,IAAI,MACR,cAAc,OAAO,KAAK,OAAO,OAAO,SAAS,MAAM,EAAE,cAAc,OAAO,OAAO,cAAc,GACrG;GAGF,MAAM,UAAU,UAAU,OAAO,QAAQ;GACzC,MAAM,gBAAgB,UAAU,OAAO,kBAAkB;GACzD,MAAM,YAAY,UAAU,SAAS;GACrC,MAAM,SAAS;IACb,GAAI,UAAU,CAAC,OAAO,IAAI,IAAI,CAAC;IAC/B,GAAI,gBAAgB,CAAC,OAAO,cAAc,IAAI,CAAC;IAC/C,GAAI,YAAY,CAAC,KAAK,IAAI,CAAC;GAC7B;GACA,MAAM,QAAQ,OAAO,SAAS,IAAI,WAAW,MAAM,IAAI,OAAO,MAAM,CAAC;GACrE,MAAM,SAAS,OAAO,MAAM,EAAE;GAC9B,OAAO,cAAc,UAAY,CAAC;GAClC,OAAO,cAAc,SAAS,CAAC;GAC/B,OAAO,cAAc,OAAO,SAAS,IAAI,gBAAgB,aAAa,CAAC;GACvE,OAAO,cAAc,YAAY,CAAC;GAClC,OAAO,cAAc,OAAO,QAAQ,EAAE;GACtC,OAAO,cAAc,MAAM,EAAE;GAC7B,OAAO,cAAc,KAAK,EAAE;GAC5B,OAAO,cAAc,OAAO,QAAQ,GAAG,EAAE;GACzC,OAAO,cAAc,gBAAgB,QAAQ,OAAO,gBAAgB,EAAE;GACtE,OAAO,cAAc,UAAU,QAAQ,OAAO,MAAM,EAAE;GACtD,OAAO,cAAc,KAAK,QAAQ,EAAE;GACpC,OAAO,cAAc,MAAM,QAAQ,EAAE;GACrC,OAAO,cAAc,GAAG,EAAE;GAC1B,OAAO,cAAc,GAAG,EAAE;GAC1B,OAAO,cAAc,GAAG,EAAE;GAC1B,OAAO,cAAc,WAAW,EAAE;GAClC,OAAO,cAAc,YAAY,QAAQ,OAAO,EAAE;GAClD,QAAQ,KAAK,QAAQ,MAAM,KAAK;EAClC;EAEA,MAAM,eAAe;EACrB,KAAK,MAAM,QAAQ,SAAS,MAAM,IAAI,IAAI;EAC1C,MAAM,cAAc,SAAS;EAC7B,MAAM,QAAQ,QAAQ;EACtB,MAAM,WAAW,UAAU,SAAS,SAAS,gBAAgB,SAAS,eAAe;EACrF,IAAI,UAAU;GACZ,MAAM,WAAW;GACjB,MAAM,SAAS,OAAO,MAAM,EAAE;GAC9B,OAAO,cAAc,WAAY,CAAC;GAClC,OAAO,iBAAiB,KAAK,CAAC;GAC9B,OAAO,cAAc,SAAS,EAAE;GAChC,OAAO,cAAc,eAAe,EAAE;GACtC,OAAO,cAAc,GAAG,EAAE;GAC1B,OAAO,cAAc,GAAG,EAAE;GAC1B,OAAO,iBAAiB,OAAO,KAAK,GAAG,EAAE;GACzC,OAAO,iBAAiB,OAAO,KAAK,GAAG,EAAE;GACzC,OAAO,iBAAiB,OAAO,WAAW,GAAG,EAAE;GAC/C,OAAO,iBAAiB,OAAO,YAAY,GAAG,EAAE;GAChD,MAAM,IAAI,MAAM;GAChB,MAAM,UAAU,OAAO,MAAM,EAAE;GAC/B,QAAQ,cAAc,WAAY,CAAC;GACnC,QAAQ,cAAc,GAAG,CAAC;GAC1B,QAAQ,iBAAiB,OAAO,QAAQ,GAAG,CAAC;GAC5C,QAAQ,cAAc,GAAG,EAAE;GAC3B,MAAM,IAAI,OAAO;EACnB;EACA,MAAM,MAAM,OAAO,MAAM,EAAE;EAC3B,IAAI,cAAc,WAAY,CAAC;EAC/B,IAAI,cAAc,GAAG,CAAC;EACtB,IAAI,cAAc,GAAG,CAAC;EACtB,IAAI,cAAc,WAAW,QAAQ,OAAO,CAAC;EAC7C,IAAI,cAAc,WAAW,QAAQ,OAAO,EAAE;EAC9C,IAAI,cAAc,WAAW,QAAQ,aAAa,EAAE;EACpD,IAAI,cAAc,WAAW,QAAQ,cAAc,EAAE;EACrD,IAAI,cAAc,GAAG,EAAE;EACvB,MAAM,IAAI,GAAG;CACf,UAAU;EACR,MAAM,OAAO,MAAM;CACrB;CACA,OAAO;AACT;;;;;;;;;;;;ACjRA,MAAM,YAAY,UAAU,IAAI;AAEhC,MAAM,OAAO,IAAI,IAAI,OAAO,QAAQ,CAAC,CAAC;AACtC,MAAM,OAAO;AACb,MAAM,MAAM,OAAO,KAAK,YAAY,QAAQ;AAC5C,MAAM,wBAA6B,IAAI,IAAI;CAAC;CAAW;CAAW;AAAS,CAAC;AAC5E,MAAM,OAAO;;AAEb,MAAM,YAAY;AAClB,MAAM,sBAAsB;AAC5B,MAAM,gBAAgB;AA0FtB,MAAM,UAAU;;AAGhB,MAAa,UAAU,UAA8B;CACnD,IAAI,MAAM;CACV,IAAI,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,QAAQ,OAAO;EACxB,SAAU,SAAS,IAAK,QAAQ;EAChC,QAAQ;EACR,OAAO,QAAQ,GAAG;GAChB,OAAO,QAAS,UAAW,OAAO,IAAM;GACxC,QAAQ;EACV;CACF;CACA,IAAI,OAAO,GAAG,OAAO,QAAS,SAAU,IAAI,OAAS;CACrD,OAAO,IAAI,SAAS,MAAM,GAAG,OAAO;CACpC,OAAO;AACT;AAEA,MAAM,YAAY,UAA8B,UAAU,OAAO,KAAK;AAEtE,MAAM,YAAY,GAAG,UAAyC;CAC5D,MAAM,OAAO,WAAW,QAAQ;CAChC,KAAK,MAAM,QAAQ,OAAO,KAAK,OAAO,IAAI;CAC1C,OAAO,KAAK,OAAO;AACrB;;AAGA,MAAM,UAAU,WAAmB,SAAyB;CAC1D,MAAM,OAAO,WAAW,MAAM,CAAC,CAC5B,OAAO,OAAO,KAAK,UAAU,QAAQ,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CACvD,OAAO,MAAM,MAAM,CAAC,CACpB,OAAO;CACV,KAAK,KAAM,KAAK,KAAM,KAAQ;CAC9B,KAAK,KAAM,KAAK,KAAM,KAAQ;CAC9B,MAAM,MAAM,KAAK,SAAS,GAAG,EAAE,CAAC,CAAC,SAAS,KAAK;CAC/C,OAAO,GAAG,IAAI,MAAM,GAAG,CAAC,EAAE,GAAG,IAAI,MAAM,GAAG,EAAE,EAAE,GAAG,IAAI,MAAM,IAAI,EAAE,EAAE,GAAG,IAAI,MAAM,IAAI,EAAE,EAAE,GAAG,IAAI,MAAM,EAAE;AACzG;;AAGA,MAAM,YAAY,OAAO,wCAAwC,IAAI;;AAGrE,MAAM,YAAY,QAAuB,SACvC,aAAa,OAAO,WAAW,GAAG,OAAO,gBAAgB,GAAG,OAAO,GAAG,GAAG,OAAO,cAAc,GAAG,MAAM,EAAE;;AAG3G,MAAM,SAAS,UAA0B,MAAM,QAAQ,YAAY,GAAG,CAAC,CAAC,KAAK;;AAG7E,MAAM,eACJ,QACA,UACW;CACX,MAAM,SAAS,MAAM,QAAQ,KAAK,SAAS,MAAM,KAAK,QAAQ,CAAC;CAC/D,MAAM,OAAO;EACX;EACA,GAAG,OAAO,KAAK,CAAC,MAAM,WAAW,GAAG,KAAK,IAAI,MAAM,KAAK,GAAG;EAC3D,mBAAmB,OAAO,MAAM;EAChC;EACA;CACF,CAAC,CAAC,KAAK,IAAI;CACX,OAAO,OAAO,OAAO;EAAC,OAAO,KAAK,MAAM,MAAM;EAAG,GAAG;EAAO;CAAG,CAAC;AACjE;;;;;AAMA,MAAM,YAAY,QAAgB,YAChC,OAAO,KACL;CACE,YAAY,OAAO,MAAM,EAAE,GAAG,aAAa,WAAW;CACtD,GAAG,QAAQ,KAAK,CAAC,MAAM,WAAW,GAAG,KAAK,IAAI,MAAM,KAAK,GAAG;CAC5D;CACA;AACF,CAAC,CAAC,KAAK,IAAI,GACX,QACF;;AAGF,MAAM,eAAyD;CAC7D,CAAC,QAAQ,MAAM;CACf,CAAC,iBAAiB,eAAe;CACjC,CAAC,iBAAiB,eAAe;CACjC,CAAC,WAAW,SAAS;CACrB,CAAC,OAAO,KAAK;CACb,CAAC,QAAQ,MAAM;CACf,CAAC,QAAQ,MAAM;AACjB;AAiBA,MAAM,QAAQ,UACZ,OAAO,UAAU,YAAY,MAAM,SAAS,IAAI,QAAQ,KAAA;;AAG1D,MAAM,SAAS,OAAgB,aAA6B;CAC1D,MAAM,KAAK,OAAO,UAAU,WAAW,KAAK,MAAM,KAAK,IAAI;CAC3D,IAAI,OAAO,SAAS,EAAE,GAAG,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY;CACzD,MAAM,QAAQ,KAAK,MAAM,QAAQ;CACjC,OAAO,OAAO,SAAS,KAAK,IAAI,IAAI,KAAK,KAAK,CAAC,CAAC,YAAY,IAAI;AAClE;AAEA,MAAM,WAAW,WAA6C;CAC5D,MAAM,UAAU,OAAO;CACvB,IAAI,OAAO,YAAY,YAAY,YAAY,MAAM,OAAO,KAAA;CAC5D,MAAM,SAAS;CACf,MAAM,MAAM,KAAK,OAAO,MAAM,KAAK,KAAK,OAAO,WAAW;CAC1D,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;CAC9B,MAAM,OAAO,KAAK,OAAO,OAAO;CAChC,MAAM,SAAS,OAAO;CACtB,OAAO;EACL;EACA,MAAM,MAAM,OAAO,eAAe,OAAO,WAAW;EACpD,QAAQ,OAAO,WAAW,YAAY,OAAO,UAAU,MAAM,IAAI,SAAS,KAAA;EAC1E,MAAM,SAAS,KAAA,KAAa,KAAK,KAAK,IAAI,IAAI,OAAO,KAAA;EACrD,aAAa,KAAK,OAAO,eAAe;EACxC,UAAU,OAAO,gBAAgB;EACjC,WAAW,OAAO,iBAAiB;EACnC,KAAK,OAAO,WAAW;EACvB,WAAW,KAAK,OAAO,aAAa;EACpC,QAAQ,KAAK,OAAO,SAAS;EAC7B,SAAS,aAAa,SAAS,CAAC,OAAO,UAAU;GAC/C,MAAM,QAAQ,KAAK,OAAO,MAAM;GAChC,OAAO,UAAU,KAAA,IAAY,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAU;EAC3D,CAAC;CACH;AACF;AAEA,MAAM,UAAU,MAAc,UAAyD;CACrF,IAAI,UAAU,KAAA,GAAW,OAAO,KAAA;CAChC,MAAM,KAAK,iBAAiB,OAAO,MAAM,QAAQ,IAAI,KAAK,MAAM,KAAK;CACrE,IAAI,CAAC,OAAO,SAAS,EAAE,GAAG,MAAM,IAAI,WAAW,GAAG,KAAK,mBAAmB,OAAO,KAAK,EAAE,EAAE;CAC1F,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY;AAClC;AAEA,MAAM,YAAY,UAAsC;CACtD,IAAI;EACF,OAAO,IAAI,IAAI,KAAK,CAAC,CAAC,SAAS,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE;CAChE,QAAQ;EACN;CACF;AACF;AAEA,MAAM,YAAY,YAA6C;CAC7D,MAAM,QAAQ,OAAO,SAAS,QAAQ,KAAK;CAC3C,MAAM,QAAQ,OAAO,SAAS,QAAQ,KAAK;CAC3C,IAAI;CACJ,IAAI,QAAQ,SAAS,KAAA,GAAW;EAC9B,OAAO,QAAQ,KAAK,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,QAAQ,SAAS,EAAE,CAAC,CAAC,QAAQ,OAAO,EAAE;EAC/E,IAAI,SAAS,MAAM,UAAU,KAAK,IAAI,GACpC,MAAM,IAAI,WAAW,sDAAsD,QAAQ,KAAK,EAAE;CAC9F;CACA,OAAO;EACL;EACA;EACA;EACA,WAAW,QAAQ,cAAc,KAAK,KAAA,IAAY,QAAQ;EAC1D,KAAK,QAAQ,QAAQ,KAAA,IAAY,KAAA,IAAY,IAAI,IAAI,QAAQ,GAAG;CAClE;AACF;AAEA,MAAM,YAAY,QAAuB,OAAc,WAAsC;CAC3F,IAAI,OAAO,UAAU,KAAA,KAAa,OAAO,cAAc,OAAO,OAAO,OAAO;CAC5E,IAAI,OAAO,UAAU,KAAA,KAAa,OAAO,eAAe,OAAO,OAAO,OAAO;CAC7E,IAAI,OAAO,cAAc,KAAA,KAAa,CAAC,MAAM,IAAI,WAAW,OAAO,SAAS,GAAG,OAAO;CACtF,IAAI,OAAO,QAAQ,KAAA,KAAa,CAAC,OAAO,IAAI,IAAI,OAAO,EAAE,GAAG,OAAO;CACnE,IAAI,OAAO,SAAS,KAAA,GAAW;EAC7B,MAAM,OAAO,SAAS,MAAM,GAAG;EAC/B,IAAI,SAAS,KAAA,KAAc,SAAS,OAAO,QAAQ,CAAC,KAAK,SAAS,IAAI,OAAO,MAAM,GACjF,OAAO;CACX;CACA,OAAO;AACT;;;;;AAMA,MAAa,eAAe,QAAuB,YAAyC;CAC1F,MAAM,SAAS,SAAS,OAAO;CAC/B,MAAM,kBAAkB,QAAQ,oBAAoB;CACpD,IAAI,OAAO,WAAW,UACpB,OAAO;EAAE,UAAU,OAAO;EAAU,OAAO,OAAO;EAAO;EAAQ;CAAgB;CAEnF,MAAM,MAAM,QAAQ,MAAM;CAC1B,IAAI,CAAC,WAAW,KAAK,KAAK,UAAU,CAAC,GACnC,MAAM,IAAI,MAAM,aAAa,IAAI,gCAAgC;CACnE,OAAO;EACL,UAAU,IAAI,WAAW,KAAK,KAAK,UAAU,GAAG,EAAE,MAAM,MAAM,CAAC;EAC/D,OAAO,IAAI,cAAc,KAAK,KAAK,OAAO,GAAG,EAAE,MAAM,MAAM,CAAC;EAC5D;EACA;CACF;AACF;;AAGA,MAAa,YACX,YAC4D;CAC5D,IAAI,OAAO,QAAQ,QAAQ,YAAY,QAAQ,IAAI,KAAK,MAAM,IAC5D,MAAM,IAAI,MAAM,8CAA8C;CAChE,MAAM,SAAS,QAAQ,QAAQ,GAAG;CAClC,IAAI,QAAQ,cAAc,QAAQ,WAAW,MAAM,GACjD,MAAM,IAAI,MAAM,GAAG,OAAO,iDAAiD;CAC7E,OAAO;EAAE;EAAQ,WAAW,GAAG,OAAO,GAAG,WAAW,EAAE;CAAM;AAC9D;;AAGA,MAAM,SAAS,OACb,OACA,SACiF;CACjF,IAAI;CACJ,IAAI;EACF,QAAQ,MAAM,MAAM,IAAI,IAAI;CAC9B,QAAQ;EACN;CACF;CACA,IAAI,UAAU,KAAA,GAAW,OAAO,KAAA;CAChC,MAAM,SAAS,SAAS,KAAK;CAC7B,OAAO,OAAO,SAAS,KAAK,MAAM,OAAO;EAAE;EAAO;CAAO,IAAI,KAAA;AAC/D;AAEA,MAAM,UAAU,gBACd,aAAa,MAAM,GAAG,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,YAAY,KAAK;;AAGtD,MAAM,cAAc,QAAuB,YACzC,OAAO,KACL,GAAG,KAAK,UAAU;CAChB,SAAS;EACP,IAAI,OAAO;EACX,QAAQ,OAAO;EACf,MAAM,OAAO;EACb,GAAI,OAAO,gBAAgB,KAAA,IAAY,EAAE,aAAa,OAAO,YAAY,IAAI,CAAC;EAC9E,GAAI,OAAO,gBAAgB,KAAA,IAAY,EAAE,aAAa,OAAO,YAAY,IAAI,CAAC;EAC9E,aAAa,OAAO;EACpB,cAAc,OAAO;EACrB,SAAS,OAAO;CAClB;CACA,QAAQ;AACV,CAAC,EAAE,KACH,MACF;;;;;;AAOF,MAAa,eAAe,OAC1B,MACA,MACA,UACA,aAC2B;CAC3B,MAAM,SAAS;EACb,UAAU;EACV,WAAW;EACX,UAAU;EACV,aAAa;EACb,UAAU;EACV,UAAU;EACV,SAAS;EACT,SAAS;CACX;CACA,MAAM,EAAE,WAAW;CACnB,MAAM,MAAM,OACV,UACkE;EAClE,MAAM,SAAS,MAAM,UAAU,KAAK;EACpC,MAAM,SAAS,KAAK;EACpB,MAAM,KAAK,MAAM,MAAM;EACvB,OAAO;GAAE;GAAQ,QAAQ,OAAO;EAAO;CACzC;CAEA,MAAM,OAAO,aAAa,WAAW,EAAE;CACvC,MAAM,QAAQ,OAAO,YACnB,OAAO,QAAQ;EACb,OAAO,OAAO;EACd,OAAO,OAAO;EACd,MAAM,OAAO;EACb,YAAY,OAAO;EACnB,KAAK,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,OAAO,IAAI;EACvD,kBAAkB,KAAK;CACzB,CAAC,CAAC,CAAC,QAAQ,GAAG,WAAW,UAAU,KAAA,CAAS,CAC9C;CACA,MAAM,SAAS;EACb,4BAA4B,QAAQ,KAAK,KAAK;EAC9C;EACA;EACA;EACA;EACA,0BAA0B,KAAK,UAAU,KAAK;CAChD;CACA,MAAM,IACJ,YACE;EACE,CAAC,aAAa,UAAU;EACxB,CAAC,kBAAkB,IAAI;EACvB,CAAC,8BAAa,IAAI,KAAK,EAAA,CAAE,YAAY,CAAC;EACtC,CAAC,iBAAiB,QAAQ;EAC1B,CAAC,gBAAgB,yBAAyB;CAC5C,GACA,CAAC,OAAO,KAAK,GAAG,OAAO,KAAK,IAAI,IAAI,QAAQ,MAAM,CAAC,CACrD,CACF;CAGA,MAAM,4BAAY,IAAI,IAGpB;CACF,MAAM,YACJ,SACA,aACS;EACT,UAAU,OAAO,OAAO;EACxB,UAAU,IAAI,SAAS,QAAQ;EAC/B,IAAI,UAAU,OAAO,WAAW,UAAU,OAAO,UAAU,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,KAAM;CACjF;CAEA,MAAM,OAAO,KAAK,SAAS,KACzB;EAAE,QAAQ;EAAS,OAAO,OAAO;CAAM,GACvC,EAAE,iBAAiB,MAAM,OAAO,WAAW,GAAG,CAChD;CACA,WAAW,MAAM,UAAU,MAAM;EAC/B,IAAI,OAAO,WAAW,WAAW,CAAC,MAAM,IAAI,OAAO,IAAI,GAAG;EAC1D,MAAM,QAAQ,QAAQ,MAAM;EAC5B,IAAI,UAAU,KAAA,KAAa,CAAC,SAAS,QAAQ,OAAO,MAAM,GAAG;EAC7D,OAAO,YAAY;EACnB,MAAM,SAA2C;GAC/C,CAAC,aAAa,MAAM,IAAI;GACxB,CAAC,mBAAmB,MAAM,GAAG;GAC7B,CAAC,oBAAoB,IAAI;EAC3B;EACA,MAAM,SAA2C,CAAC,CAAC,+BAA+B,OAAO,EAAE,CAAC;EAC5F,IAAI;EACJ,IAAI,UAAkC;GAAE,QAAQ;GAAQ,QAAQ;EAAU;EAE1E,IAAI,OAAO,SAAS,aAAa,MAAM,SAAS,KAAA,GAAW;GAEzD,MAAM,WAAW,MAAM,cAAc,KAAA,IAAY,KAAA,IAAY,UAAU,IAAI,MAAM,SAAS;GAC1F,MAAM,OAAO,SAAS,KAAK,CACzB,GAAI,MAAM,gBAAgB,KAAA,IAAY,CAAC,CAAC,gBAAgB,MAAM,WAAW,CAAU,IAAI,CAAC,GACxF,GAAG,MAAM,OACX,CAAC;GACD,MAAM,SAAS,SAAS,OAAO,KAAK,MAAM,MAAM,KAAK,CAAC;GACtD,OAAO,SAAS,QAAQ,SAAS;GACjC,MAAM,KAAK,MAAM,IACf,YACE;IACE,CAAC,aAAa,SAAS;IACvB,CAAC,kBAAkB,IAAI;IACvB,GAAG;IACH,CAAC,gBAAgB,mBAAmB;IACpC,GAAI,aAAa,KAAA,IAAY,CAAC,CAAC,kBAAkB,SAAS,EAAE,CAAU,IAAI,CAAC;IAC3E,CAAC,6BAA6B,UAAU,OAAO,MAAM,GAAG;IACxD,GAAI,aAAa,KAAA,IAAY,CAAC,CAAC,uBAAuB,SAAS,IAAI,CAAU,IAAI,CAAC;IAClF,CAAC,uBAAuB,MAAM;IAC9B,CAAC,qBAAqB,SAAS,SAAS,IAAI,CAAC,CAAC;IAC9C,GAAG;IACH,CAAC,+BAA+B,SAAS;IACzC,CAAC,gBAAgB,aAAa;GAChC,GACA,CAAC,IAAI,CACP,CACF;GACA,OAAO,YAAY;GACnB,UAAU;IAAE,QAAQ;IAAW,cAAc;GAAU;GACvD,MAAM,WAAW;IACf,MAAM;IACN,KAAK,MAAM;IACX,MAAM,MAAM;IACZ,QAAQ;IACR,MAAM;IACN;IACA,GAAG;IACH,SAAS,OAAO;IAChB,MAAM;GACR,CAAC;EACH,OAAO,IAAI,OAAO,SAAS,aAAa,MAAM,SAAS,KAAA,GAAW;GAChE,MAAM,OAAO,MAAM,OAAO,KAAK,OAAO,MAAM,IAAI;GAChD,IAAI,SAAS,KAAA,GAAW;IACtB,OAAO,WAAW;IAClB,UAAU;KAAE,QAAQ;KAAQ,QAAQ;IAAe;GACrD,OAAO,IAAI,MAAM,UAAU;IAEzB,MAAM,SAAS,SAAS,KAAK,MAAM;IACnC,OAAO,SAAS,QAAQ,YAAY;IACpC,MAAM,IACJ,YACE;KACE,CAAC,aAAa,YAAY;KAC1B,CAAC,kBAAkB,IAAI;KACvB,GAAG;KACH,CAAC,uBAAuB,MAAM;KAC9B,CAAC,qBAAqB,MAAM;KAC5B,GAAG;KACH,CAAC,gBAAgB,MAAM,eAAe,0BAA0B;IAClE,GACA,CAAC,KAAK,KAAK,CACb,CACF;IACA,OAAO,eAAe;IACtB,UAAU;KAAE,QAAQ;KAAc,MAAM;IAAW;GACrD,OAAO,IAAI,MAAM,WAAW,KAAA,KAAa,MAAM,UAAU,OAAO,MAAM,UAAU,KAAK;IACnF,MAAM,OAAO,SAAS,MAAM,QAAQ;KAClC,GAAI,MAAM,gBAAgB,KAAA,IACtB,CAAC,CAAC,gBAAgB,MAAM,WAAW,CAAU,IAC7C,CAAC;KACL,CAAC,kBAAkB,OAAO,KAAK,MAAM,MAAM,CAAC;KAC5C,GAAG,MAAM;IACX,CAAC;IACD,MAAM,SAAS,SAAS,KAAK,MAAM;IACnC,OAAO,SAAS,QAAQ,UAAU;IAElC,MAAM,YAAY,MAAM,MAAM,eAAe,MAAM,YAAY,WAAW,KAAA;IAC1E,MAAM,KAAK,MAAM,IACf,YACE;KACE,CAAC,aAAa,UAAU;KACxB,CAAC,kBAAkB,IAAI;KACvB,GAAG;KACH,CAAC,uBAAuB,MAAM;KAC9B,CAAC,qBAAqB,SAAS,SAAS,MAAM,KAAK,KAAK,CAAC,CAAC;KAC1D,GAAI,cAAc,KAAA,IAAY,CAAC,CAAC,kBAAkB,SAAS,CAAU,IAAI,CAAC;KAC1E,GAAG;KACH,CAAC,+BAA+B,SAAS;KACzC,CAAC,gBAAgB,aAAa;IAChC,GACA,CAAC,MAAM,KAAK,KAAK,CACnB,CACF;IACA,OAAO,aAAa;IACpB,UAAU;KAAE,QAAQ;KAAY,cAAc;KAAW,MAAM;IAAU;IACzE,SAAS,OAAO,IAAI;KAAE,IAAI;KAAM,MAAM,MAAM;KAAM,KAAK,MAAM;IAAI,CAAC;IAClE,MAAM,WAAW;KACf,MAAM;KACN,KAAK,MAAM;KACX,MAAM,MAAM;KACZ,QAAQ,MAAM;KACd,MAAM,OAAO,MAAM,WAAW;KAC9B;KACA,GAAG;KACH,SAAS,OAAO;KAChB,MAAM,MAAM,WAAW,OAAO,MAAM,WAAW;IACjD,CAAC;GACH;EACF;EAEA,IAAI,SAAS,KAAA,GAAW;GACtB,IAAI,QAAQ,cAAc,WAAW,OAAO,YAAY;GACxD,IAAI,CAAC,KAAK,iBAAiB;EAC7B;EACA,MAAM,QAAQ,WAAW,QAAQ,OAAO;EACxC,MAAM,IACJ,YACE;GACE,CAAC,aAAa,UAAU;GACxB,CAAC,kBAAkB,SAAS,QAAQ,UAAU,CAAC;GAC/C,GAAG;GACH,GAAI,SAAS,KAAA,IAAY,CAAC,CAAC,sBAAsB,IAAI,CAAU,IAAI,CAAC;GACpE,CAAC,qBAAqB,SAAS,SAAS,KAAK,CAAC,CAAC;GAC/C,GAAG;GACH,CAAC,gBAAgB,kBAAkB;EACrC,GACA,CAAC,KAAK,CACR,CACF;EACA,OAAO,YAAY;CACrB;CACA,OAAO;AACT;;;;;;;AAQA,MAAa,QAAQ,UAA0B;CAC7C,IAAI;CACJ,IAAI;EACF,MAAM,IAAI,IAAI,KAAK;CACrB,QAAQ;EACN,OAAO,MAAM,YAAY;CAC3B;CACA,MAAM,OAAO,IAAI,SACd,YAAY,CAAC,CACb,QAAQ,OAAO,EAAE,CAAC,CAClB,QAAQ,aAAa,EAAE;CAC1B,MAAM,MAAM,KAAK,WAAW,GAAG,IAAI,OAAO,KAAK,MAAM,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,GAAG;CAC5E,IAAI,OAAO,IAAI,SAAS,YAAY;CACpC,IAAI,KAAK,SAAS,KAAK,KAAK,SAAS,GAAG,GAAG,OAAO,KAAK,MAAM,GAAG,EAAE;CAClE,MAAM,QAAQ,IAAI,OAAO,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG;CAC1F,OAAO,GAAG,MAAM,IAAI,SAAS,KAAK,KAAK,IAAI,IAAI,OAAO,GAAG,OAAO,UAAU,KAAK,KAAK,IAAI;AAC1F;;AAGA,MAAa,YAAY,QAAwB,aAC/C,GAAG,KAAK,OAAO,GAAG,EAAE,GAAG,OAAO,KAAK,MAAM,GAAG,EAAE,CAAC,CAAC,QAAQ,OAAO,EAAE,EAAE,GAAG,KAAK,UAAU;CACnF,KAAK,OAAO;CACZ,MAAM,OAAO;CACb,QAAQ,OAAO,OAAO,MAAM;CAC5B,QAAQ,OAAO;CACf,QAAQ,OAAO,OAAO,MAAM;CAC5B,QAAQ,OAAO,OAAO,MAAM;CAC5B;AACF,CAAC;;;;;;;;;;AAWH,MAAa,aAAa,OACxB,QACA,YACwB;CACxB,MAAM,OAAO,YAAY,QAAQ,OAAO;CACxC,MAAM,EAAE,QAAQ,cAAc,SAAS,OAAO;CAC9C,MAAM,OAAO,MAAM,YAAY,OAAO,SAAS;CAC/C,IAAI;EACF,MAAM,SAAS,MAAM,aAAa,MAAM,MAAM,SAAS,MAAM,CAAC;EAC9D,MAAM,OAAO,MAAM,KAAK,MAAM;EAC9B,MAAM,OAAO,WAAW,MAAM;EAC9B,OAAO;GAAE,QAAQ;GAAQ,KAAK;GAAQ,OAAO,KAAK;GAAM,GAAG;EAAO;CACpE,SAAS,OAAO;EACd,MAAM,KAAK,QAAQ;EACnB,MAAM,GAAG,WAAW,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;EAC1D,MAAM;CACR;AACF"}
+{"version":3,"file":"warc-B47Bdjzk.mjs","names":[],"sources":["../src/zip.ts","../src/warc.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport { createReadStream, createWriteStream } from \"node:fs\";\nimport type { FileHandle } from \"node:fs/promises\";\nimport { open, stat } from \"node:fs/promises\";\nimport { pipeline } from \"node:stream/promises\";\nimport * as zlib from \"node:zlib\";\n\n/*\n * This module writes a ZIP file with node:zlib only. The WACZ export uses it. It is not part of the\n * API of the package.\n *\n * - A member is stored (method 0) or deflated (method 8).\n * - The writer knows the CRC-32 and the sizes of each member before it writes the member. Thus the\n *   local headers hold the true values, and the file has no data descriptors.\n * - The writer adds the Zip64 records only when a size, an offset or the count of members is too\n *   large for the fields of 32 bits. The option `zip64: \"always\"` adds them in each case, for a test.\n */\n\nconst TABLE: Uint32Array = (() => {\n  const table = new Uint32Array(256);\n  for (let n = 0; n < 256; n += 1) {\n    let c = n;\n    for (let k = 0; k < 8; k += 1) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;\n    table[n] = c >>> 0;\n  }\n  return table;\n})();\n\n/** This function calculates the CRC-32 of ISO 3309 with a table. It is the fallback of `crc32`. */\nexport const tableCrc32 = (bytes: Uint8Array, value = 0): number => {\n  let crc = ~value >>> 0;\n  for (const byte of bytes) crc = TABLE[(crc ^ byte) & 0xff]! ^ (crc >>> 8);\n  return ~crc >>> 0;\n};\n\nconst native = (zlib as { readonly crc32?: (data: Uint8Array, value?: number) => number }).crc32;\n\n/** This function calculates the CRC-32 of the bytes. It continues from `value`. It uses `zlib.crc32` when Node.js has it. */\nexport const crc32 = (bytes: Uint8Array, value = 0): number =>\n  native === undefined ? tableCrc32(bytes, value) : native(bytes, value);\n\n/** These are the facts of a file that a tally gives at its close. */\nexport interface Tallied {\n  readonly path: string;\n  readonly size: number;\n  readonly crc: number;\n  /** This value is the SHA-256 of the bytes, as hexadecimal digits. */\n  readonly sha256: string;\n}\n\n/** This function writes the whole buffer. A write can write less than the buffer, and the loop writes the rest. */\nexport const writeAll = async (handle: FileHandle, bytes: Uint8Array): Promise<void> => {\n  let at = 0;\n  while (at < bytes.length) {\n    const { bytesWritten } = await handle.write(bytes, at, bytes.length - at);\n    at += bytesWritten;\n  }\n};\n\n/** A tally writes a new file in sequence. It counts the bytes and calculates their CRC-32 and SHA-256. */\nexport class TalliedFile {\n  private readonly hash = createHash(\"sha256\");\n  private crcValue = 0;\n  private count = 0;\n\n  private constructor(\n    readonly path: string,\n    private readonly handle: FileHandle,\n  ) {}\n\n  /** This method makes a new file. It fails when the file exists. */\n  static async create(path: string): Promise<TalliedFile> {\n    return new TalliedFile(path, await open(path, \"wx\", 0o600));\n  }\n\n  /** This value is the count of bytes that the tally wrote. It is the offset of the next write. */\n  get size(): number {\n    return this.count;\n  }\n\n  async write(bytes: Uint8Array): Promise<void> {\n    await writeAll(this.handle, bytes);\n    this.hash.update(bytes);\n    this.crcValue = crc32(bytes, this.crcValue);\n    this.count += bytes.length;\n  }\n\n  /** This method closes the file and gives its facts. */\n  async close(): Promise<Tallied> {\n    await this.handle.close();\n    return {\n      path: this.path,\n      size: this.count,\n      crc: this.crcValue,\n      sha256: this.hash.digest(\"hex\"),\n    };\n  }\n\n  /** This method closes the file after an error. It gives no facts. */\n  async abandon(): Promise<void> {\n    await this.handle.close().catch(() => undefined);\n  }\n}\n\n/** This function deflates a file into a new file (raw deflate, as ZIP needs it). It gives the size of the new file. */\nexport const deflateFile = async (from: string, to: string): Promise<number> => {\n  await pipeline(\n    createReadStream(from),\n    zlib.createDeflateRaw(),\n    createWriteStream(to, { flags: \"wx\", mode: 0o600 }),\n  );\n  return (await stat(to)).size;\n};\n\n/** A member of a ZIP file. Its bytes are in a file or in memory, as the ZIP stores them. */\nexport interface ZipMember {\n  /** This value is the path of the member in the ZIP file. */\n  readonly name: string;\n  /** The value 0 stores the bytes. The value 8 holds deflated bytes. */\n  readonly method: 0 | 8;\n  /** This value is the CRC-32 of the bytes before compression. */\n  readonly crc: number;\n  /** This value is the count of bytes before compression. */\n  readonly size: number;\n  /** This value is the count of bytes in the ZIP file. */\n  readonly compressedSize: number;\n  /** This file holds the bytes as the ZIP stores them. */\n  readonly file?: string | undefined;\n  /** These bytes are the member as the ZIP stores them, when `file` is not given. */\n  readonly data?: Uint8Array | undefined;\n}\n\n/** This function makes a member from bytes in memory. A member of method 8 holds the bytes deflated. */\nexport const memberOf = (name: string, bytes: Uint8Array, method: 0 | 8 = 8): ZipMember => {\n  const data = method === 8 ? zlib.deflateRawSync(bytes) : bytes;\n  return {\n    name,\n    method,\n    crc: crc32(bytes),\n    size: bytes.length,\n    compressedSize: data.length,\n    data,\n  };\n};\n\n/** These options change a ZIP file. */\nexport interface ZipOptions {\n  /** This time goes into each member, in the MS-DOS format of ZIP (UTC, to 2 seconds). */\n  readonly date: Date;\n  /** The value `auto` (the default) adds the Zip64 records only when they are necessary. */\n  readonly zip64?: \"auto\" | \"always\" | undefined;\n}\n\nconst MAX32 = 0xffffffff;\nconst MAX16 = 0xffff;\nconst VERSION_ZIP = 20;\nconst VERSION_ZIP64 = 45;\n/** The member was made on Unix (3), with version 4.5 of the specification. */\nconst MADE_BY = (3 << 8) | VERSION_ZIP64;\n/** A plain file that its owner can read and write, and the others can read (`0o100644`). */\nconst FILE_MODE = (0o100644 << 16) >>> 0;\n/** Bit 11 of the flags tells that the name is UTF-8. */\nconst UTF8_NAMES = 0x0800;\n\nconst dosTime = (date: Date): { readonly time: number; readonly day: number } => {\n  const year = Math.min(Math.max(date.getUTCFullYear(), 1980), 2107);\n  return {\n    time: (date.getUTCHours() << 11) | (date.getUTCMinutes() << 5) | (date.getUTCSeconds() >> 1),\n    day: ((year - 1980) << 9) | ((date.getUTCMonth() + 1) << 5) | date.getUTCDate(),\n  };\n};\n\n/** This function makes the Zip64 extra field (id 0x0001) with these values of 64 bits. */\nconst zip64Extra = (values: readonly number[]): Buffer => {\n  const extra = Buffer.alloc(4 + 8 * values.length);\n  extra.writeUInt16LE(0x0001, 0);\n  extra.writeUInt16LE(8 * values.length, 2);\n  values.forEach((value, at) => extra.writeBigUInt64LE(BigInt(value), 4 + 8 * at));\n  return extra;\n};\n\n/**\n * This function writes a ZIP file with these members, in this sequence. It writes a new file and\n * fails when the file exists. It gives the size of the ZIP file.\n */\nexport const writeZip = async (\n  target: string,\n  members: readonly ZipMember[],\n  options: ZipOptions,\n): Promise<number> => {\n  const always = options.zip64 === \"always\";\n  const { time, day } = dosTime(options.date);\n  const handle = await open(target, \"wx\", 0o600);\n  let offset = 0;\n  const put = async (bytes: Uint8Array): Promise<void> => {\n    await writeAll(handle, bytes);\n    offset += bytes.length;\n  };\n  try {\n    const central: Buffer[] = [];\n    for (const member of members) {\n      const name = Buffer.from(member.name, \"utf8\");\n      const start = offset;\n      const bigSizes = always || member.size >= MAX32 || member.compressedSize >= MAX32;\n      const localExtra = bigSizes\n        ? zip64Extra([member.size, member.compressedSize])\n        : Buffer.alloc(0);\n      const local = Buffer.alloc(30);\n      local.writeUInt32LE(0x04034b50, 0);\n      local.writeUInt16LE(bigSizes ? VERSION_ZIP64 : VERSION_ZIP, 4);\n      local.writeUInt16LE(UTF8_NAMES, 6);\n      local.writeUInt16LE(member.method, 8);\n      local.writeUInt16LE(time, 10);\n      local.writeUInt16LE(day, 12);\n      local.writeUInt32LE(member.crc >>> 0, 14);\n      local.writeUInt32LE(bigSizes ? MAX32 : member.compressedSize, 18);\n      local.writeUInt32LE(bigSizes ? MAX32 : member.size, 22);\n      local.writeUInt16LE(name.length, 26);\n      local.writeUInt16LE(localExtra.length, 28);\n      await put(local);\n      await put(name);\n      await put(localExtra);\n      const before = offset;\n      if (member.file !== undefined) {\n        for await (const chunk of createReadStream(member.file) as AsyncIterable<Buffer>)\n          await put(chunk);\n      } else await put(member.data ?? new Uint8Array());\n      if (offset - before !== member.compressedSize)\n        throw new Error(\n          `the member ${member.name} has ${String(offset - before)} bytes, not ${String(member.compressedSize)}`,\n        );\n\n      // the central header names only the values that are too large, in this sequence\n      const bigSize = always || member.size >= MAX32;\n      const bigCompressed = always || member.compressedSize >= MAX32;\n      const bigOffset = always || start >= MAX32;\n      const values = [\n        ...(bigSize ? [member.size] : []),\n        ...(bigCompressed ? [member.compressedSize] : []),\n        ...(bigOffset ? [start] : []),\n      ];\n      const extra = values.length > 0 ? zip64Extra(values) : Buffer.alloc(0);\n      const header = Buffer.alloc(46);\n      header.writeUInt32LE(0x02014b50, 0);\n      header.writeUInt16LE(MADE_BY, 4);\n      header.writeUInt16LE(values.length > 0 ? VERSION_ZIP64 : VERSION_ZIP, 6);\n      header.writeUInt16LE(UTF8_NAMES, 8);\n      header.writeUInt16LE(member.method, 10);\n      header.writeUInt16LE(time, 12);\n      header.writeUInt16LE(day, 14);\n      header.writeUInt32LE(member.crc >>> 0, 16);\n      header.writeUInt32LE(bigCompressed ? MAX32 : member.compressedSize, 20);\n      header.writeUInt32LE(bigSize ? MAX32 : member.size, 24);\n      header.writeUInt16LE(name.length, 28);\n      header.writeUInt16LE(extra.length, 30);\n      header.writeUInt16LE(0, 32);\n      header.writeUInt16LE(0, 34);\n      header.writeUInt16LE(0, 36);\n      header.writeUInt32LE(FILE_MODE, 38);\n      header.writeUInt32LE(bigOffset ? MAX32 : start, 42);\n      central.push(header, name, extra);\n    }\n\n    const centralStart = offset;\n    for (const part of central) await put(part);\n    const centralSize = offset - centralStart;\n    const count = members.length;\n    const zip64End = always || count >= MAX16 || centralStart >= MAX32 || centralSize >= MAX32;\n    if (zip64End) {\n      const recordAt = offset;\n      const record = Buffer.alloc(56);\n      record.writeUInt32LE(0x06064b50, 0);\n      record.writeBigUInt64LE(44n, 4);\n      record.writeUInt16LE(MADE_BY, 12);\n      record.writeUInt16LE(VERSION_ZIP64, 14);\n      record.writeUInt32LE(0, 16);\n      record.writeUInt32LE(0, 20);\n      record.writeBigUInt64LE(BigInt(count), 24);\n      record.writeBigUInt64LE(BigInt(count), 32);\n      record.writeBigUInt64LE(BigInt(centralSize), 40);\n      record.writeBigUInt64LE(BigInt(centralStart), 48);\n      await put(record);\n      const locator = Buffer.alloc(20);\n      locator.writeUInt32LE(0x07064b50, 0);\n      locator.writeUInt32LE(0, 4);\n      locator.writeBigUInt64LE(BigInt(recordAt), 8);\n      locator.writeUInt32LE(1, 16);\n      await put(locator);\n    }\n    const end = Buffer.alloc(22);\n    end.writeUInt32LE(0x06054b50, 0);\n    end.writeUInt16LE(0, 4);\n    end.writeUInt16LE(0, 6);\n    end.writeUInt16LE(zip64End ? MAX16 : count, 8);\n    end.writeUInt16LE(zip64End ? MAX16 : count, 10);\n    end.writeUInt32LE(zip64End ? MAX32 : centralSize, 12);\n    end.writeUInt32LE(zip64End ? MAX32 : centralStart, 16);\n    end.writeUInt16LE(0, 20);\n    await put(end);\n  } finally {\n    await handle.close();\n  }\n  return offset;\n};\n","import { createHash, randomUUID } from \"node:crypto\";\nimport { existsSync } from \"node:fs\";\nimport { rename, rm } from \"node:fs/promises\";\nimport { STATUS_CODES } from \"node:http\";\nimport { basename, join, resolve } from \"node:path\";\nimport { promisify } from \"node:util\";\nimport { gzip } from \"node:zlib\";\nimport type { BlobStore } from \"./blobs.ts\";\nimport { FileBlobStore } from \"./blobs.ts\";\nimport type { CaptureReader } from \"./log.ts\";\nimport { CaptureLog } from \"./log.ts\";\nimport type { CaptureRecord } from \"./record.ts\";\nimport { BOT_PAGE, VERSION } from \"./version.ts\";\nimport { TalliedFile } from \"./zip.ts\";\n\n/*\n * This module writes the capture log as WARC/1.1 (ISO 28500:2017). The README tells what goes into\n * the archive and why. These are the records:\n *\n * - One `warcinfo` record names the software and the filters of the export.\n * - A capture with a body gives a `response` record. Its HTTP header block is rebuilt from the\n *   headers that the capture kept, and its payload is the stored body.\n * - A 304 gives a `revisit` record with the profile `server-not-modified`.\n * - A page that the headless browser rendered gives a `conversion` record: the page after\n *   JavaScript, not the body that the server sent.\n * - Each of them has a `metadata` record (`WARC-Concurrent-To`) with the JSON of the capture.\n *\n * Each record is one gzip member. The export reads the log as a stream and holds one body at a time.\n */\n\nconst gzipAsync = promisify(gzip);\n\nconst HOME = new URL(\"../\", BOT_PAGE).href;\nconst CRLF = \"\\r\\n\";\nconst END = Buffer.from(\"\\r\\n\\r\\n\", \"latin1\");\nconst KINDS: ReadonlySet<string> = new Set([\"capture\", \"revisit\", \"skipped\"]);\nconst HASH = /^[0-9a-f]{64}$/;\n/** The revisit records name their original from this count of recent responses. */\nconst ORIGINALS = 65_536;\nconst SERVER_NOT_MODIFIED = \"http://netpreserve.org/warc/1.1/revisit/server-not-modified\";\nconst HTTP_RESPONSE = \"application/http; msgtype=response\";\n\n/** The source of an export: the folder of a store, or the capture log and the blob store of a runtime. */\nexport type ArchiveSource =\n  string | { readonly captures: CaptureReader; readonly blobs: BlobStore };\n\n/** These filters select the captures of an export. Each filter that you give makes the selection smaller. */\nexport interface ArchiveFilter {\n  /** The export holds the captures received at this time or later. */\n  readonly since?: string | Date | undefined;\n  /** The export holds the captures received before this time. */\n  readonly until?: string | Date | undefined;\n  /** The export holds the captures of this host and of its subdomains, for example `example.com`. */\n  readonly host?: string | undefined;\n  /** The export holds the captures whose URL starts with this text. */\n  readonly urlPrefix?: string | undefined;\n  /** The export holds only the captures with these ids. */\n  readonly ids?: readonly string[] | undefined;\n}\n\n/** These options control an export. Only `out` is necessary. */\nexport interface ArchiveOptions extends ArchiveFilter {\n  /** This file gets the archive. The export writes a temporary file and gives it this name at the end. */\n  readonly out: string;\n  /**\n   * When this value is true, each capture with no body (a refusal, a skip of robots.txt, an error, a\n   * redirect) gets a `metadata` record. The default is false: the export leaves them out.\n   */\n  readonly includeRefusals?: boolean | undefined;\n  /** When this value is true, the export replaces a file that exists. The default is false: the export fails. */\n  readonly overwrite?: boolean | undefined;\n}\n\n/** These counts tell what an export wrote. */\nexport interface ArchiveCounts {\n  /** This value is the count of capture records that the filters selected. */\n  readonly captures: number;\n  readonly responses: number;\n  readonly revisits: number;\n  /** This value is the count of rendered pages (`conversion` records). */\n  readonly conversions: number;\n  readonly metadata: number;\n  /** This value is the count of captures with no body: refusals, skips, errors and redirects. */\n  readonly bodiless: number;\n  /** This value is the count of captures whose body is not in the blob store, or does not agree with its hash. */\n  readonly missing: number;\n  /** This value is the count of damaged lines of the capture log. */\n  readonly damaged: number;\n}\n\n/** This is the result of `exportWarc`. */\nexport interface WarcResult extends ArchiveCounts {\n  readonly format: \"warc\";\n  /** This value is the absolute path of the archive. */\n  readonly out: string;\n  readonly bytes: number;\n}\n\n/** These are the facts of a `response` or `revisit` record that the index of a WACZ file needs. */\nexport interface ArchivedRecord {\n  readonly type: \"response\" | \"revisit\";\n  readonly url: string;\n  readonly date: string;\n  readonly status: number;\n  readonly mime: string;\n  readonly digest: string;\n  readonly offset: number;\n  readonly length: number;\n  readonly capture: string;\n  /** This value is true for a page of the page list: a 200 response that is not a robots.txt file. */\n  readonly page: boolean;\n}\n\n/** A plan is a valid filter and an open source. An export makes it before it writes a file. */\nexport interface ArchivePlan {\n  readonly captures: CaptureReader;\n  readonly blobs: BlobStore;\n  readonly filter: ArchiveSelection;\n  readonly includeRefusals: boolean;\n}\n\n/** This is a filter after the checks: times in ISO 8601, a lowercase host and a set of ids. */\nexport interface ArchiveSelection {\n  readonly since?: string | undefined;\n  readonly until?: string | undefined;\n  readonly host?: string | undefined;\n  readonly urlPrefix?: string | undefined;\n  readonly ids?: ReadonlySet<string> | undefined;\n}\n\nconst RFC4648 = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567\";\n\n/** This function writes bytes in base 32 (RFC 4648), with capitals and padding. WARC tools write SHA-256 digests in this form. */\nexport const base32 = (bytes: Uint8Array): string => {\n  let out = \"\";\n  let value = 0;\n  let bits = 0;\n  for (const byte of bytes) {\n    value = ((value << 8) | byte) & 0xffff;\n    bits += 8;\n    while (bits >= 5) {\n      out += RFC4648[(value >>> (bits - 5)) & 31]!;\n      bits -= 5;\n    }\n  }\n  if (bits > 0) out += RFC4648[(value << (5 - bits)) & 31]!;\n  while (out.length % 8 !== 0) out += \"=\";\n  return out;\n};\n\nconst digestOf = (bytes: Uint8Array): string => `sha256:${base32(bytes)}`;\n\nconst sha256Of = (...parts: readonly Uint8Array[]): Buffer => {\n  const hash = createHash(\"sha256\");\n  for (const part of parts) hash.update(part);\n  return hash.digest();\n};\n\n/** This function gives a version 5 UUID (RFC 9562): the SHA-1 of a namespace and a name. */\nconst uuidV5 = (namespace: string, name: string): string => {\n  const hash = createHash(\"sha1\")\n    .update(Buffer.from(namespace.replace(/-/g, \"\"), \"hex\"))\n    .update(name, \"utf8\")\n    .digest();\n  hash[6] = (hash[6]! & 0x0f) | 0x50;\n  hash[8] = (hash[8]! & 0x3f) | 0x80;\n  const hex = hash.subarray(0, 16).toString(\"hex\");\n  return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;\n};\n\n/** The namespace of the record ids: a UUID of the URL of the site, in the URL namespace of RFC 9562. */\nconst NAMESPACE = uuidV5(\"6ba7b811-9dad-11d1-80b4-00c04fd430c8\", HOME);\n\n/** The id of a record of a capture is the same at each export: it comes from the hash of the capture. */\nconst recordId = (record: CaptureRecord, type: string): string =>\n  `<urn:uuid:${uuidV5(NAMESPACE, `${record.content_hash || `${record.id}@${record.received_at}`}/${type}`)}>`;\n\n/** A field of a header holds no line break. */\nconst clean = (value: string): string => value.replace(/[\\r\\n]+/g, \" \").trim();\n\n/** This function gives one record: the WARC header, the block and the two line breaks at the end. */\nconst recordBytes = (\n  fields: ReadonlyArray<readonly [string, string]>,\n  block: readonly Uint8Array[],\n): Buffer => {\n  const length = block.reduce((sum, part) => sum + part.length, 0);\n  const head = [\n    \"WARC/1.1\",\n    ...fields.map(([name, value]) => `${name}: ${clean(value)}`),\n    `Content-Length: ${String(length)}`,\n    \"\",\n    \"\",\n  ].join(CRLF);\n  return Buffer.concat([Buffer.from(head, \"utf8\"), ...block, END]);\n};\n\n/**\n * This function gives an HTTP status line and header block. HTTP headers are Latin-1 bytes, and\n * undici gives each byte as one character. Thus the block gives back the bytes that the server sent.\n */\nconst httpHead = (status: number, headers: ReadonlyArray<readonly [string, string]>): Buffer =>\n  Buffer.from(\n    [\n      `HTTP/1.1 ${String(status)} ${STATUS_CODES[status] ?? \"Unknown\"}`,\n      ...headers.map(([name, value]) => `${name}: ${clean(value)}`),\n      \"\",\n      \"\",\n    ].join(CRLF),\n    \"latin1\",\n  );\n\n/** These fields of a capture keep a header of the response. */\nconst KEPT_HEADERS: ReadonlyArray<readonly [string, string]> = [\n  [\"etag\", \"ETag\"],\n  [\"last_modified\", \"Last-Modified\"],\n  [\"cache_control\", \"Cache-Control\"],\n  [\"expires\", \"Expires\"],\n  [\"age\", \"Age\"],\n  [\"date\", \"Date\"],\n  [\"vary\", \"Vary\"],\n];\n\n/** These are the facts of a fetch record that the export reads. */\ninterface Facts {\n  readonly url: string;\n  readonly date: string;\n  readonly status: number | undefined;\n  readonly blob: string | undefined;\n  readonly contentType: string | undefined;\n  readonly rendered: boolean;\n  readonly truncated: boolean;\n  readonly cut: boolean;\n  readonly revisitOf: string | undefined;\n  readonly recipe: string | undefined;\n  readonly headers: ReadonlyArray<readonly [string, string]>;\n}\n\nconst text = (value: unknown): string | undefined =>\n  typeof value === \"string\" && value.length > 0 ? value : undefined;\n\n/** This function gives the time of a capture in ISO 8601, with milliseconds. WARC/1.1 permits them. */\nconst isoOf = (value: unknown, fallback: string): string => {\n  const ms = typeof value === \"string\" ? Date.parse(value) : Number.NaN;\n  if (Number.isFinite(ms)) return new Date(ms).toISOString();\n  const other = Date.parse(fallback);\n  return Number.isFinite(other) ? new Date(other).toISOString() : fallback;\n};\n\nconst factsOf = (record: CaptureRecord): Facts | undefined => {\n  const payload = record.payload;\n  if (typeof payload !== \"object\" || payload === null) return undefined;\n  const fields = payload as Readonly<Record<string, unknown>>;\n  const url = text(fields[\"url\"]) ?? text(record.external_id);\n  if (url === undefined) return undefined;\n  const blob = text(fields[\"blob\"]);\n  const status = fields[\"status\"];\n  return {\n    url,\n    date: isoOf(fields[\"fetched_at\"], record.received_at),\n    status: typeof status === \"number\" && Number.isInteger(status) ? status : undefined,\n    blob: blob !== undefined && HASH.test(blob) ? blob : undefined,\n    contentType: text(fields[\"content_type\"]),\n    rendered: fields[\"rendered\"] === true,\n    truncated: fields[\"truncated\"] === true,\n    cut: fields[\"cut\"] === true,\n    revisitOf: text(fields[\"revisit_of\"]),\n    recipe: text(fields[\"recipe\"]),\n    headers: KEPT_HEADERS.flatMap(([field, name]) => {\n      const value = text(fields[field]);\n      return value === undefined ? [] : [[name, value] as const];\n    }),\n  };\n};\n\nconst timeOf = (name: string, value: string | Date | undefined): string | undefined => {\n  if (value === undefined) return undefined;\n  const ms = value instanceof Date ? value.getTime() : Date.parse(value);\n  if (!Number.isFinite(ms)) throw new RangeError(`${name} is not a time: \"${String(value)}\"`);\n  return new Date(ms).toISOString();\n};\n\nconst hostName = (value: string): string | undefined => {\n  try {\n    return new URL(value).hostname.toLowerCase().replace(/\\.$/, \"\");\n  } catch {\n    return undefined;\n  }\n};\n\nconst filterOf = (options: ArchiveFilter): ArchiveSelection => {\n  const since = timeOf(\"since\", options.since);\n  const until = timeOf(\"until\", options.until);\n  let host: string | undefined;\n  if (options.host !== undefined) {\n    host = options.host.trim().toLowerCase().replace(/^\\*\\./, \"\").replace(/\\.$/, \"\");\n    if (host === \"\" || /[/:@\\s]/.test(host))\n      throw new RangeError(`host is a host name, for example example.com, not \"${options.host}\"`);\n  }\n  return {\n    since,\n    until,\n    host,\n    urlPrefix: options.urlPrefix === \"\" ? undefined : options.urlPrefix,\n    ids: options.ids === undefined ? undefined : new Set(options.ids),\n  };\n};\n\nconst selected = (record: CaptureRecord, facts: Facts, filter: ArchiveSelection): boolean => {\n  if (filter.since !== undefined && record.received_at < filter.since) return false;\n  if (filter.until !== undefined && record.received_at >= filter.until) return false;\n  if (filter.urlPrefix !== undefined && !facts.url.startsWith(filter.urlPrefix)) return false;\n  if (filter.ids !== undefined && !filter.ids.has(record.id)) return false;\n  if (filter.host !== undefined) {\n    const host = hostName(facts.url);\n    if (host === undefined || (host !== filter.host && !host.endsWith(`.${filter.host}`)))\n      return false;\n  }\n  return true;\n};\n\n/**\n * This function checks the filters and opens the source. It writes nothing. A folder must hold a\n * capture log, and the export does not make a store.\n */\nexport const planArchive = (source: ArchiveSource, options: ArchiveOptions): ArchivePlan => {\n  const filter = filterOf(options);\n  const includeRefusals = options.includeRefusals === true;\n  if (typeof source !== \"string\") {\n    return { captures: source.captures, blobs: source.blobs, filter, includeRefusals };\n  }\n  const dir = resolve(source);\n  if (!existsSync(join(dir, \"captures\")))\n    throw new Error(`the store ${dir} has no capture log (captures/)`);\n  return {\n    captures: new CaptureLog(join(dir, \"captures\"), { sync: false }),\n    blobs: new FileBlobStore(join(dir, \"blobs\"), { sync: false }),\n    filter,\n    includeRefusals,\n  };\n};\n\n/** This function gives the path of the archive and of its temporary file. It fails when the archive exists, unless `overwrite` is true. */\nexport const outputOf = (\n  options: ArchiveOptions,\n): { readonly target: string; readonly temporary: string } => {\n  if (typeof options.out !== \"string\" || options.out.trim() === \"\")\n    throw new Error(\"an export needs out: the file of the archive\");\n  const target = resolve(options.out);\n  if (options.overwrite !== true && existsSync(target))\n    throw new Error(`${target} exists: give a different file, or set overwrite`);\n  return { target, temporary: `${target}.${randomUUID()}.tmp` };\n};\n\n/** This function reads a body from the blob store. It gives no body when the bytes do not agree with the hash. */\nconst bodyOf = async (\n  blobs: BlobStore,\n  hash: string,\n): Promise<{ readonly bytes: Uint8Array; readonly digest: Buffer } | undefined> => {\n  let bytes: Uint8Array | undefined;\n  try {\n    bytes = await blobs.get(hash);\n  } catch {\n    return undefined;\n  }\n  if (bytes === undefined) return undefined;\n  const digest = sha256Of(bytes);\n  return digest.toString(\"hex\") === hash ? { bytes, digest } : undefined;\n};\n\nconst mimeOf = (contentType: string | undefined): string =>\n  contentType?.split(\";\")[0]?.trim().toLowerCase() || \"-\";\n\n/** This function gives the JSON of the metadata record of a capture. The labels and the scope of the application stay out. */\nconst metadataOf = (record: CaptureRecord, written: Readonly<Record<string, string>>): Buffer =>\n  Buffer.from(\n    `${JSON.stringify({\n      capture: {\n        id: record.id,\n        source: record.source,\n        kind: record.kind,\n        ...(record.external_id !== undefined ? { external_id: record.external_id } : {}),\n        ...(record.occurred_at !== undefined ? { occurred_at: record.occurred_at } : {}),\n        received_at: record.received_at,\n        content_hash: record.content_hash,\n        payload: record.payload,\n      },\n      export: written,\n    })}\\n`,\n    \"utf8\",\n  );\n\n/**\n * This function writes the records of an archive into a file: the `warcinfo` record, then the\n * records of each capture that the plan selects. It gives the facts of each `response` and\n * `revisit` record to `onRecord`, for the index of a WACZ file.\n */\nexport const writeArchive = async (\n  plan: ArchivePlan,\n  file: TalliedFile,\n  filename: string,\n  onRecord?: (record: ArchivedRecord) => Promise<void>,\n): Promise<ArchiveCounts> => {\n  const counts = {\n    captures: 0,\n    responses: 0,\n    revisits: 0,\n    conversions: 0,\n    metadata: 0,\n    bodiless: 0,\n    missing: 0,\n    damaged: 0,\n  };\n  const { filter } = plan;\n  const put = async (\n    bytes: Buffer,\n  ): Promise<{ readonly offset: number; readonly length: number }> => {\n    const member = await gzipAsync(bytes);\n    const offset = file.size;\n    await file.write(member);\n    return { offset, length: member.length };\n  };\n\n  const info = `<urn:uuid:${randomUUID()}>`;\n  const shown = Object.fromEntries(\n    Object.entries({\n      since: filter.since,\n      until: filter.until,\n      host: filter.host,\n      url_prefix: filter.urlPrefix,\n      ids: filter.ids === undefined ? undefined : filter.ids.size,\n      include_refusals: plan.includeRefusals,\n    }).filter(([, value]) => value !== undefined),\n  );\n  const fields = [\n    `software: polite-crawler/${VERSION} (+${HOME})`,\n    \"format: WARC File Format 1.1\",\n    \"conformsTo: https://iipc.github.io/warc-specifications/specifications/warc-format/warc-1.1/\",\n    \"robots: obey\",\n    \"description: An export of the capture log of polite-crawler. The HTTP headers of each response are rebuilt from the headers that the capture kept.\",\n    `polite-crawler-filter: ${JSON.stringify(shown)}`,\n  ];\n  await put(\n    recordBytes(\n      [\n        [\"WARC-Type\", \"warcinfo\"],\n        [\"WARC-Record-ID\", info],\n        [\"WARC-Date\", new Date().toISOString()],\n        [\"WARC-Filename\", filename],\n        [\"Content-Type\", \"application/warc-fields\"],\n      ],\n      [Buffer.from(`${fields.join(CRLF)}${CRLF}`, \"utf8\")],\n    ),\n  );\n\n  // a revisit names its original when the original is one of the recent responses of the export\n  const originals = new Map<\n    string,\n    { readonly id: string; readonly date: string; readonly url: string }\n  >();\n  const remember = (\n    capture: string,\n    original: { readonly id: string; readonly date: string; readonly url: string },\n  ): void => {\n    originals.delete(capture);\n    originals.set(capture, original);\n    if (originals.size > ORIGINALS) originals.delete(originals.keys().next().value!);\n  };\n\n  const scan = plan.captures.scan(\n    { source: \"fetch\", since: filter.since },\n    { onDamaged: () => void (counts.damaged += 1) },\n  );\n  for await (const record of scan) {\n    if (record.source !== \"fetch\" || !KINDS.has(record.kind)) continue;\n    const facts = factsOf(record);\n    if (facts === undefined || !selected(record, facts, filter)) continue;\n    counts.captures += 1;\n    const common: Array<readonly [string, string]> = [\n      [\"WARC-Date\", facts.date],\n      [\"WARC-Target-URI\", facts.url],\n      [\"WARC-Warcinfo-ID\", info],\n    ];\n    const vendor: Array<readonly [string, string]> = [[\"WARC-Polite-Crawler-Capture\", record.id]];\n    let main: string | undefined;\n    let written: Record<string, string> = { record: \"none\", reason: \"no body\" };\n\n    if (record.kind === \"revisit\" && facts.blob !== undefined) {\n      // the 304 confirmed the stored body, so the payload digest is the digest of that body\n      const original = facts.revisitOf === undefined ? undefined : originals.get(facts.revisitOf);\n      const head = httpHead(304, [\n        ...(facts.contentType !== undefined ? [[\"Content-Type\", facts.contentType] as const] : []),\n        ...facts.headers,\n      ]);\n      const digest = digestOf(Buffer.from(facts.blob, \"hex\"));\n      main = recordId(record, \"revisit\");\n      const at = await put(\n        recordBytes(\n          [\n            [\"WARC-Type\", \"revisit\"],\n            [\"WARC-Record-ID\", main],\n            ...common,\n            [\"WARC-Profile\", SERVER_NOT_MODIFIED],\n            ...(original !== undefined ? [[\"WARC-Refers-To\", original.id] as const] : []),\n            [\"WARC-Refers-To-Target-URI\", original?.url ?? facts.url],\n            ...(original !== undefined ? [[\"WARC-Refers-To-Date\", original.date] as const] : []),\n            [\"WARC-Payload-Digest\", digest],\n            [\"WARC-Block-Digest\", digestOf(sha256Of(head))],\n            ...vendor,\n            [\"WARC-Polite-Crawler-Headers\", \"rebuilt\"],\n            [\"Content-Type\", HTTP_RESPONSE],\n          ],\n          [head],\n        ),\n      );\n      counts.revisits += 1;\n      written = { record: \"revisit\", http_headers: \"rebuilt\" };\n      await onRecord?.({\n        type: \"revisit\",\n        url: facts.url,\n        date: facts.date,\n        status: 304,\n        mime: \"warc/revisit\",\n        digest,\n        ...at,\n        capture: record.id,\n        page: false,\n      });\n    } else if (record.kind === \"capture\" && facts.blob !== undefined) {\n      const body = await bodyOf(plan.blobs, facts.blob);\n      if (body === undefined) {\n        counts.missing += 1;\n        written = { record: \"none\", reason: \"body missing\" };\n      } else if (facts.rendered) {\n        // the page after JavaScript is a conversion of the page, not the body that the server sent\n        const digest = digestOf(body.digest);\n        main = recordId(record, \"conversion\");\n        await put(\n          recordBytes(\n            [\n              [\"WARC-Type\", \"conversion\"],\n              [\"WARC-Record-ID\", main],\n              ...common,\n              [\"WARC-Payload-Digest\", digest],\n              [\"WARC-Block-Digest\", digest],\n              ...vendor,\n              [\"Content-Type\", facts.contentType ?? \"text/html; charset=utf-8\"],\n            ],\n            [body.bytes],\n          ),\n        );\n        counts.conversions += 1;\n        written = { record: \"conversion\", body: \"rendered\" };\n      } else if (facts.status !== undefined && facts.status >= 100 && facts.status <= 999) {\n        const head = httpHead(facts.status, [\n          ...(facts.contentType !== undefined\n            ? [[\"Content-Type\", facts.contentType] as const]\n            : []),\n          [\"Content-Length\", String(body.bytes.length)],\n          ...facts.headers,\n        ]);\n        const digest = digestOf(body.digest);\n        main = recordId(record, \"response\");\n        // a body that the crawler cut at its limit, or that ended early, says so\n        const truncated = facts.cut ? \"disconnect\" : facts.truncated ? \"length\" : undefined;\n        const at = await put(\n          recordBytes(\n            [\n              [\"WARC-Type\", \"response\"],\n              [\"WARC-Record-ID\", main],\n              ...common,\n              [\"WARC-Payload-Digest\", digest],\n              [\"WARC-Block-Digest\", digestOf(sha256Of(head, body.bytes))],\n              ...(truncated !== undefined ? [[\"WARC-Truncated\", truncated] as const] : []),\n              ...vendor,\n              [\"WARC-Polite-Crawler-Headers\", \"rebuilt\"],\n              [\"Content-Type\", HTTP_RESPONSE],\n            ],\n            [head, body.bytes],\n          ),\n        );\n        counts.responses += 1;\n        written = { record: \"response\", http_headers: \"rebuilt\", body: \"decoded\" };\n        remember(record.id, { id: main, date: facts.date, url: facts.url });\n        await onRecord?.({\n          type: \"response\",\n          url: facts.url,\n          date: facts.date,\n          status: facts.status,\n          mime: mimeOf(facts.contentType),\n          digest,\n          ...at,\n          capture: record.id,\n          page: facts.status === 200 && facts.recipe !== \"robots.txt\",\n        });\n      }\n    }\n\n    if (main === undefined) {\n      if (written[\"reason\"] === \"no body\") counts.bodiless += 1;\n      if (!plan.includeRefusals) continue;\n    }\n    const block = metadataOf(record, written);\n    await put(\n      recordBytes(\n        [\n          [\"WARC-Type\", \"metadata\"],\n          [\"WARC-Record-ID\", recordId(record, \"metadata\")],\n          ...common,\n          ...(main !== undefined ? [[\"WARC-Concurrent-To\", main] as const] : []),\n          [\"WARC-Block-Digest\", digestOf(sha256Of(block))],\n          ...vendor,\n          [\"Content-Type\", \"application/json\"],\n        ],\n        [block],\n      ),\n    );\n    counts.metadata += 1;\n  }\n  return counts;\n};\n\n/**\n * This function gives the SURT key of a URL, as `pywb` and the `surt` library make it with their\n * defaults. The host is reversed with commas and has no `www.` and no default port. The path\n * and the query are lowercase, the path has no slash at the end, and the parts of the query are\n * sorted.\n */\nexport const surt = (value: string): string => {\n  let url: URL;\n  try {\n    url = new URL(value);\n  } catch {\n    return value.toLowerCase();\n  }\n  const host = url.hostname\n    .toLowerCase()\n    .replace(/\\.$/, \"\")\n    .replace(/^www\\d*\\./, \"\");\n  const key = host.startsWith(\"[\") ? host : host.split(\".\").reverse().join(\",\");\n  let path = url.pathname.toLowerCase();\n  if (path.length > 1 && path.endsWith(\"/\")) path = path.slice(0, -1);\n  const query = url.search.slice(1).toLowerCase().split(\"&\").filter(Boolean).sort().join(\"&\");\n  return `${key}${url.port === \"\" ? \"\" : `:${url.port}`})${path}${query === \"\" ? \"\" : `?${query}`}`;\n};\n\n/** This function gives the line of a record in a CDXJ index: the SURT key, the time to the second, and JSON. */\nexport const cdxjLine = (record: ArchivedRecord, filename: string): string =>\n  `${surt(record.url)} ${record.date.slice(0, 19).replace(/\\D/g, \"\")} ${JSON.stringify({\n    url: record.url,\n    mime: record.mime,\n    status: String(record.status),\n    digest: record.digest,\n    length: String(record.length),\n    offset: String(record.offset),\n    filename,\n  })}`;\n\n/**\n * This function writes the captures of a store as a WARC/1.1 file, with one gzip member for each\n * record (`.warc.gz`). It reads the capture log as a stream, so its memory does not grow with the\n * store. It writes a temporary file and gives it the name `out` at the end, so a failed export\n * leaves no part of a file.\n *\n * The source is the folder of a store, or the capture log and the blob store of a runtime\n * (`crawler.runtime`). The export only reads the store.\n */\nexport const exportWarc = async (\n  source: ArchiveSource,\n  options: ArchiveOptions,\n): Promise<WarcResult> => {\n  const plan = planArchive(source, options);\n  const { target, temporary } = outputOf(options);\n  const file = await TalliedFile.create(temporary);\n  try {\n    const counts = await writeArchive(plan, file, basename(target));\n    const done = await file.close();\n    await rename(temporary, target);\n    return { format: \"warc\", out: target, bytes: done.size, ...counts };\n  } catch (error) {\n    await file.abandon();\n    await rm(temporary, { force: true }).catch(() => undefined);\n    throw error;\n  }\n};\n"],"mappings":";;;;;;;;;;;;;AAkBA,MAAM,eAA4B;CAChC,MAAM,wBAAQ,IAAI,YAAY,GAAG;CACjC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK,GAAG;EAC/B,IAAI,IAAI;EACR,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,IAAI,IAAI,aAAc,MAAM,IAAK,MAAM;EAC1E,MAAM,KAAK,MAAM;CACnB;CACA,OAAO;AACT,EAAA,CAAG;;AAGH,MAAa,cAAc,OAAmB,QAAQ,MAAc;CAClE,IAAI,MAAM,CAAC,UAAU;CACrB,KAAK,MAAM,QAAQ,OAAO,MAAM,OAAO,MAAM,QAAQ,OAAU,QAAQ;CACvE,OAAO,CAAC,QAAQ;AAClB;AAEA,MAAM,SAAU,KAA2E;;AAG3F,MAAa,SAAS,OAAmB,QAAQ,MAC/C,WAAW,KAAA,IAAY,WAAW,OAAO,KAAK,IAAI,OAAO,OAAO,KAAK;;AAYvE,MAAa,WAAW,OAAO,QAAoB,UAAqC;CACtF,IAAI,KAAK;CACT,OAAO,KAAK,MAAM,QAAQ;EACxB,MAAM,EAAE,iBAAiB,MAAM,OAAO,MAAM,OAAO,IAAI,MAAM,SAAS,EAAE;EACxE,MAAM;CACR;AACF;;AAGA,IAAa,cAAb,MAAa,YAAY;CAMZ;CACQ;CANnB,OAAwB,WAAW,QAAQ;CAC3C,WAAmB;CACnB,QAAgB;CAEhB,YACE,MACA,QACA;EAFS,KAAA,OAAA;EACQ,KAAA,SAAA;CAChB;;CAGH,aAAa,OAAO,MAAoC;EACtD,OAAO,IAAI,YAAY,MAAM,MAAM,KAAK,MAAM,MAAM,GAAK,CAAC;CAC5D;;CAGA,IAAI,OAAe;EACjB,OAAO,KAAK;CACd;CAEA,MAAM,MAAM,OAAkC;EAC5C,MAAM,SAAS,KAAK,QAAQ,KAAK;EACjC,KAAK,KAAK,OAAO,KAAK;EACtB,KAAK,WAAW,MAAM,OAAO,KAAK,QAAQ;EAC1C,KAAK,SAAS,MAAM;CACtB;;CAGA,MAAM,QAA0B;EAC9B,MAAM,KAAK,OAAO,MAAM;EACxB,OAAO;GACL,MAAM,KAAK;GACX,MAAM,KAAK;GACX,KAAK,KAAK;GACV,QAAQ,KAAK,KAAK,OAAO,KAAK;EAChC;CACF;;CAGA,MAAM,UAAyB;EAC7B,MAAM,KAAK,OAAO,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;CACjD;AACF;;AAGA,MAAa,cAAc,OAAO,MAAc,OAAgC;CAC9E,MAAM,SACJ,iBAAiB,IAAI,GACrB,KAAK,iBAAiB,GACtB,kBAAkB,IAAI;EAAE,OAAO;EAAM,MAAM;CAAM,CAAC,CACpD;CACA,QAAQ,MAAM,KAAK,EAAE,EAAA,CAAG;AAC1B;;AAqBA,MAAa,YAAY,MAAc,OAAmB,SAAgB,MAAiB;CACzF,MAAM,OAAO,WAAW,IAAI,KAAK,eAAe,KAAK,IAAI;CACzD,OAAO;EACL;EACA;EACA,KAAK,MAAM,KAAK;EAChB,MAAM,MAAM;EACZ,gBAAgB,KAAK;EACrB;CACF;AACF;AAUA,MAAM,QAAQ;AACd,MAAM,QAAQ;AACd,MAAM,cAAc;AACpB,MAAM,gBAAgB;;AAEtB,MAAM,UAAU;;AAEhB,MAAM,YAAa,SAAY,OAAQ;;AAEvC,MAAM,aAAa;AAEnB,MAAM,WAAW,SAAgE;CAC/E,MAAM,OAAO,KAAK,IAAI,KAAK,IAAI,KAAK,eAAe,GAAG,IAAI,GAAG,IAAI;CACjE,OAAO;EACL,MAAO,KAAK,YAAY,KAAK,KAAO,KAAK,cAAc,KAAK,IAAM,KAAK,cAAc,KAAK;EAC1F,KAAO,OAAO,QAAS,IAAO,KAAK,YAAY,IAAI,KAAM,IAAK,KAAK,WAAW;CAChF;AACF;;AAGA,MAAM,cAAc,WAAsC;CACxD,MAAM,QAAQ,OAAO,MAAM,IAAI,IAAI,OAAO,MAAM;CAChD,MAAM,cAAc,GAAQ,CAAC;CAC7B,MAAM,cAAc,IAAI,OAAO,QAAQ,CAAC;CACxC,OAAO,SAAS,OAAO,OAAO,MAAM,iBAAiB,OAAO,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;CAC/E,OAAO;AACT;;;;;AAMA,MAAa,WAAW,OACtB,QACA,SACA,YACoB;CACpB,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,EAAE,MAAM,QAAQ,QAAQ,QAAQ,IAAI;CAC1C,MAAM,SAAS,MAAM,KAAK,QAAQ,MAAM,GAAK;CAC7C,IAAI,SAAS;CACb,MAAM,MAAM,OAAO,UAAqC;EACtD,MAAM,SAAS,QAAQ,KAAK;EAC5B,UAAU,MAAM;CAClB;CACA,IAAI;EACF,MAAM,UAAoB,CAAC;EAC3B,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,OAAO,OAAO,KAAK,OAAO,MAAM,MAAM;GAC5C,MAAM,QAAQ;GACd,MAAM,WAAW,UAAU,OAAO,QAAQ,SAAS,OAAO,kBAAkB;GAC5E,MAAM,aAAa,WACf,WAAW,CAAC,OAAO,MAAM,OAAO,cAAc,CAAC,IAC/C,OAAO,MAAM,CAAC;GAClB,MAAM,QAAQ,OAAO,MAAM,EAAE;GAC7B,MAAM,cAAc,UAAY,CAAC;GACjC,MAAM,cAAc,WAAW,gBAAgB,aAAa,CAAC;GAC7D,MAAM,cAAc,YAAY,CAAC;GACjC,MAAM,cAAc,OAAO,QAAQ,CAAC;GACpC,MAAM,cAAc,MAAM,EAAE;GAC5B,MAAM,cAAc,KAAK,EAAE;GAC3B,MAAM,cAAc,OAAO,QAAQ,GAAG,EAAE;GACxC,MAAM,cAAc,WAAW,QAAQ,OAAO,gBAAgB,EAAE;GAChE,MAAM,cAAc,WAAW,QAAQ,OAAO,MAAM,EAAE;GACtD,MAAM,cAAc,KAAK,QAAQ,EAAE;GACnC,MAAM,cAAc,WAAW,QAAQ,EAAE;GACzC,MAAM,IAAI,KAAK;GACf,MAAM,IAAI,IAAI;GACd,MAAM,IAAI,UAAU;GACpB,MAAM,SAAS;GACf,IAAI,OAAO,SAAS,KAAA,GAClB,WAAW,MAAM,SAAS,iBAAiB,OAAO,IAAI,GACpD,MAAM,IAAI,KAAK;QACZ,MAAM,IAAI,OAAO,wBAAQ,IAAI,WAAW,CAAC;GAChD,IAAI,SAAS,WAAW,OAAO,gBAC7B,MAAM,IAAI,MACR,cAAc,OAAO,KAAK,OAAO,OAAO,SAAS,MAAM,EAAE,cAAc,OAAO,OAAO,cAAc,GACrG;GAGF,MAAM,UAAU,UAAU,OAAO,QAAQ;GACzC,MAAM,gBAAgB,UAAU,OAAO,kBAAkB;GACzD,MAAM,YAAY,UAAU,SAAS;GACrC,MAAM,SAAS;IACb,GAAI,UAAU,CAAC,OAAO,IAAI,IAAI,CAAC;IAC/B,GAAI,gBAAgB,CAAC,OAAO,cAAc,IAAI,CAAC;IAC/C,GAAI,YAAY,CAAC,KAAK,IAAI,CAAC;GAC7B;GACA,MAAM,QAAQ,OAAO,SAAS,IAAI,WAAW,MAAM,IAAI,OAAO,MAAM,CAAC;GACrE,MAAM,SAAS,OAAO,MAAM,EAAE;GAC9B,OAAO,cAAc,UAAY,CAAC;GAClC,OAAO,cAAc,SAAS,CAAC;GAC/B,OAAO,cAAc,OAAO,SAAS,IAAI,gBAAgB,aAAa,CAAC;GACvE,OAAO,cAAc,YAAY,CAAC;GAClC,OAAO,cAAc,OAAO,QAAQ,EAAE;GACtC,OAAO,cAAc,MAAM,EAAE;GAC7B,OAAO,cAAc,KAAK,EAAE;GAC5B,OAAO,cAAc,OAAO,QAAQ,GAAG,EAAE;GACzC,OAAO,cAAc,gBAAgB,QAAQ,OAAO,gBAAgB,EAAE;GACtE,OAAO,cAAc,UAAU,QAAQ,OAAO,MAAM,EAAE;GACtD,OAAO,cAAc,KAAK,QAAQ,EAAE;GACpC,OAAO,cAAc,MAAM,QAAQ,EAAE;GACrC,OAAO,cAAc,GAAG,EAAE;GAC1B,OAAO,cAAc,GAAG,EAAE;GAC1B,OAAO,cAAc,GAAG,EAAE;GAC1B,OAAO,cAAc,WAAW,EAAE;GAClC,OAAO,cAAc,YAAY,QAAQ,OAAO,EAAE;GAClD,QAAQ,KAAK,QAAQ,MAAM,KAAK;EAClC;EAEA,MAAM,eAAe;EACrB,KAAK,MAAM,QAAQ,SAAS,MAAM,IAAI,IAAI;EAC1C,MAAM,cAAc,SAAS;EAC7B,MAAM,QAAQ,QAAQ;EACtB,MAAM,WAAW,UAAU,SAAS,SAAS,gBAAgB,SAAS,eAAe;EACrF,IAAI,UAAU;GACZ,MAAM,WAAW;GACjB,MAAM,SAAS,OAAO,MAAM,EAAE;GAC9B,OAAO,cAAc,WAAY,CAAC;GAClC,OAAO,iBAAiB,KAAK,CAAC;GAC9B,OAAO,cAAc,SAAS,EAAE;GAChC,OAAO,cAAc,eAAe,EAAE;GACtC,OAAO,cAAc,GAAG,EAAE;GAC1B,OAAO,cAAc,GAAG,EAAE;GAC1B,OAAO,iBAAiB,OAAO,KAAK,GAAG,EAAE;GACzC,OAAO,iBAAiB,OAAO,KAAK,GAAG,EAAE;GACzC,OAAO,iBAAiB,OAAO,WAAW,GAAG,EAAE;GAC/C,OAAO,iBAAiB,OAAO,YAAY,GAAG,EAAE;GAChD,MAAM,IAAI,MAAM;GAChB,MAAM,UAAU,OAAO,MAAM,EAAE;GAC/B,QAAQ,cAAc,WAAY,CAAC;GACnC,QAAQ,cAAc,GAAG,CAAC;GAC1B,QAAQ,iBAAiB,OAAO,QAAQ,GAAG,CAAC;GAC5C,QAAQ,cAAc,GAAG,EAAE;GAC3B,MAAM,IAAI,OAAO;EACnB;EACA,MAAM,MAAM,OAAO,MAAM,EAAE;EAC3B,IAAI,cAAc,WAAY,CAAC;EAC/B,IAAI,cAAc,GAAG,CAAC;EACtB,IAAI,cAAc,GAAG,CAAC;EACtB,IAAI,cAAc,WAAW,QAAQ,OAAO,CAAC;EAC7C,IAAI,cAAc,WAAW,QAAQ,OAAO,EAAE;EAC9C,IAAI,cAAc,WAAW,QAAQ,aAAa,EAAE;EACpD,IAAI,cAAc,WAAW,QAAQ,cAAc,EAAE;EACrD,IAAI,cAAc,GAAG,EAAE;EACvB,MAAM,IAAI,GAAG;CACf,UAAU;EACR,MAAM,OAAO,MAAM;CACrB;CACA,OAAO;AACT;;;;;;;;;;;;ACjRA,MAAM,YAAY,UAAU,IAAI;AAEhC,MAAM,OAAO,IAAI,IAAI,OAAO,QAAQ,CAAC,CAAC;AACtC,MAAM,OAAO;AACb,MAAM,MAAM,OAAO,KAAK,YAAY,QAAQ;AAC5C,MAAM,wBAA6B,IAAI,IAAI;CAAC;CAAW;CAAW;AAAS,CAAC;AAC5E,MAAM,OAAO;;AAEb,MAAM,YAAY;AAClB,MAAM,sBAAsB;AAC5B,MAAM,gBAAgB;AA0FtB,MAAM,UAAU;;AAGhB,MAAa,UAAU,UAA8B;CACnD,IAAI,MAAM;CACV,IAAI,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,QAAQ,OAAO;EACxB,SAAU,SAAS,IAAK,QAAQ;EAChC,QAAQ;EACR,OAAO,QAAQ,GAAG;GAChB,OAAO,QAAS,UAAW,OAAO,IAAM;GACxC,QAAQ;EACV;CACF;CACA,IAAI,OAAO,GAAG,OAAO,QAAS,SAAU,IAAI,OAAS;CACrD,OAAO,IAAI,SAAS,MAAM,GAAG,OAAO;CACpC,OAAO;AACT;AAEA,MAAM,YAAY,UAA8B,UAAU,OAAO,KAAK;AAEtE,MAAM,YAAY,GAAG,UAAyC;CAC5D,MAAM,OAAO,WAAW,QAAQ;CAChC,KAAK,MAAM,QAAQ,OAAO,KAAK,OAAO,IAAI;CAC1C,OAAO,KAAK,OAAO;AACrB;;AAGA,MAAM,UAAU,WAAmB,SAAyB;CAC1D,MAAM,OAAO,WAAW,MAAM,CAAC,CAC5B,OAAO,OAAO,KAAK,UAAU,QAAQ,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CACvD,OAAO,MAAM,MAAM,CAAC,CACpB,OAAO;CACV,KAAK,KAAM,KAAK,KAAM,KAAQ;CAC9B,KAAK,KAAM,KAAK,KAAM,KAAQ;CAC9B,MAAM,MAAM,KAAK,SAAS,GAAG,EAAE,CAAC,CAAC,SAAS,KAAK;CAC/C,OAAO,GAAG,IAAI,MAAM,GAAG,CAAC,EAAE,GAAG,IAAI,MAAM,GAAG,EAAE,EAAE,GAAG,IAAI,MAAM,IAAI,EAAE,EAAE,GAAG,IAAI,MAAM,IAAI,EAAE,EAAE,GAAG,IAAI,MAAM,EAAE;AACzG;;AAGA,MAAM,YAAY,OAAO,wCAAwC,IAAI;;AAGrE,MAAM,YAAY,QAAuB,SACvC,aAAa,OAAO,WAAW,GAAG,OAAO,gBAAgB,GAAG,OAAO,GAAG,GAAG,OAAO,cAAc,GAAG,MAAM,EAAE;;AAG3G,MAAM,SAAS,UAA0B,MAAM,QAAQ,YAAY,GAAG,CAAC,CAAC,KAAK;;AAG7E,MAAM,eACJ,QACA,UACW;CACX,MAAM,SAAS,MAAM,QAAQ,KAAK,SAAS,MAAM,KAAK,QAAQ,CAAC;CAC/D,MAAM,OAAO;EACX;EACA,GAAG,OAAO,KAAK,CAAC,MAAM,WAAW,GAAG,KAAK,IAAI,MAAM,KAAK,GAAG;EAC3D,mBAAmB,OAAO,MAAM;EAChC;EACA;CACF,CAAC,CAAC,KAAK,IAAI;CACX,OAAO,OAAO,OAAO;EAAC,OAAO,KAAK,MAAM,MAAM;EAAG,GAAG;EAAO;CAAG,CAAC;AACjE;;;;;AAMA,MAAM,YAAY,QAAgB,YAChC,OAAO,KACL;CACE,YAAY,OAAO,MAAM,EAAE,GAAG,aAAa,WAAW;CACtD,GAAG,QAAQ,KAAK,CAAC,MAAM,WAAW,GAAG,KAAK,IAAI,MAAM,KAAK,GAAG;CAC5D;CACA;AACF,CAAC,CAAC,KAAK,IAAI,GACX,QACF;;AAGF,MAAM,eAAyD;CAC7D,CAAC,QAAQ,MAAM;CACf,CAAC,iBAAiB,eAAe;CACjC,CAAC,iBAAiB,eAAe;CACjC,CAAC,WAAW,SAAS;CACrB,CAAC,OAAO,KAAK;CACb,CAAC,QAAQ,MAAM;CACf,CAAC,QAAQ,MAAM;AACjB;AAiBA,MAAM,QAAQ,UACZ,OAAO,UAAU,YAAY,MAAM,SAAS,IAAI,QAAQ,KAAA;;AAG1D,MAAM,SAAS,OAAgB,aAA6B;CAC1D,MAAM,KAAK,OAAO,UAAU,WAAW,KAAK,MAAM,KAAK,IAAI;CAC3D,IAAI,OAAO,SAAS,EAAE,GAAG,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY;CACzD,MAAM,QAAQ,KAAK,MAAM,QAAQ;CACjC,OAAO,OAAO,SAAS,KAAK,IAAI,IAAI,KAAK,KAAK,CAAC,CAAC,YAAY,IAAI;AAClE;AAEA,MAAM,WAAW,WAA6C;CAC5D,MAAM,UAAU,OAAO;CACvB,IAAI,OAAO,YAAY,YAAY,YAAY,MAAM,OAAO,KAAA;CAC5D,MAAM,SAAS;CACf,MAAM,MAAM,KAAK,OAAO,MAAM,KAAK,KAAK,OAAO,WAAW;CAC1D,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;CAC9B,MAAM,OAAO,KAAK,OAAO,OAAO;CAChC,MAAM,SAAS,OAAO;CACtB,OAAO;EACL;EACA,MAAM,MAAM,OAAO,eAAe,OAAO,WAAW;EACpD,QAAQ,OAAO,WAAW,YAAY,OAAO,UAAU,MAAM,IAAI,SAAS,KAAA;EAC1E,MAAM,SAAS,KAAA,KAAa,KAAK,KAAK,IAAI,IAAI,OAAO,KAAA;EACrD,aAAa,KAAK,OAAO,eAAe;EACxC,UAAU,OAAO,gBAAgB;EACjC,WAAW,OAAO,iBAAiB;EACnC,KAAK,OAAO,WAAW;EACvB,WAAW,KAAK,OAAO,aAAa;EACpC,QAAQ,KAAK,OAAO,SAAS;EAC7B,SAAS,aAAa,SAAS,CAAC,OAAO,UAAU;GAC/C,MAAM,QAAQ,KAAK,OAAO,MAAM;GAChC,OAAO,UAAU,KAAA,IAAY,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAU;EAC3D,CAAC;CACH;AACF;AAEA,MAAM,UAAU,MAAc,UAAyD;CACrF,IAAI,UAAU,KAAA,GAAW,OAAO,KAAA;CAChC,MAAM,KAAK,iBAAiB,OAAO,MAAM,QAAQ,IAAI,KAAK,MAAM,KAAK;CACrE,IAAI,CAAC,OAAO,SAAS,EAAE,GAAG,MAAM,IAAI,WAAW,GAAG,KAAK,mBAAmB,OAAO,KAAK,EAAE,EAAE;CAC1F,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY;AAClC;AAEA,MAAM,YAAY,UAAsC;CACtD,IAAI;EACF,OAAO,IAAI,IAAI,KAAK,CAAC,CAAC,SAAS,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE;CAChE,QAAQ;EACN;CACF;AACF;AAEA,MAAM,YAAY,YAA6C;CAC7D,MAAM,QAAQ,OAAO,SAAS,QAAQ,KAAK;CAC3C,MAAM,QAAQ,OAAO,SAAS,QAAQ,KAAK;CAC3C,IAAI;CACJ,IAAI,QAAQ,SAAS,KAAA,GAAW;EAC9B,OAAO,QAAQ,KAAK,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,QAAQ,SAAS,EAAE,CAAC,CAAC,QAAQ,OAAO,EAAE;EAC/E,IAAI,SAAS,MAAM,UAAU,KAAK,IAAI,GACpC,MAAM,IAAI,WAAW,sDAAsD,QAAQ,KAAK,EAAE;CAC9F;CACA,OAAO;EACL;EACA;EACA;EACA,WAAW,QAAQ,cAAc,KAAK,KAAA,IAAY,QAAQ;EAC1D,KAAK,QAAQ,QAAQ,KAAA,IAAY,KAAA,IAAY,IAAI,IAAI,QAAQ,GAAG;CAClE;AACF;AAEA,MAAM,YAAY,QAAuB,OAAc,WAAsC;CAC3F,IAAI,OAAO,UAAU,KAAA,KAAa,OAAO,cAAc,OAAO,OAAO,OAAO;CAC5E,IAAI,OAAO,UAAU,KAAA,KAAa,OAAO,eAAe,OAAO,OAAO,OAAO;CAC7E,IAAI,OAAO,cAAc,KAAA,KAAa,CAAC,MAAM,IAAI,WAAW,OAAO,SAAS,GAAG,OAAO;CACtF,IAAI,OAAO,QAAQ,KAAA,KAAa,CAAC,OAAO,IAAI,IAAI,OAAO,EAAE,GAAG,OAAO;CACnE,IAAI,OAAO,SAAS,KAAA,GAAW;EAC7B,MAAM,OAAO,SAAS,MAAM,GAAG;EAC/B,IAAI,SAAS,KAAA,KAAc,SAAS,OAAO,QAAQ,CAAC,KAAK,SAAS,IAAI,OAAO,MAAM,GACjF,OAAO;CACX;CACA,OAAO;AACT;;;;;AAMA,MAAa,eAAe,QAAuB,YAAyC;CAC1F,MAAM,SAAS,SAAS,OAAO;CAC/B,MAAM,kBAAkB,QAAQ,oBAAoB;CACpD,IAAI,OAAO,WAAW,UACpB,OAAO;EAAE,UAAU,OAAO;EAAU,OAAO,OAAO;EAAO;EAAQ;CAAgB;CAEnF,MAAM,MAAM,QAAQ,MAAM;CAC1B,IAAI,CAAC,WAAW,KAAK,KAAK,UAAU,CAAC,GACnC,MAAM,IAAI,MAAM,aAAa,IAAI,gCAAgC;CACnE,OAAO;EACL,UAAU,IAAI,WAAW,KAAK,KAAK,UAAU,GAAG,EAAE,MAAM,MAAM,CAAC;EAC/D,OAAO,IAAI,cAAc,KAAK,KAAK,OAAO,GAAG,EAAE,MAAM,MAAM,CAAC;EAC5D;EACA;CACF;AACF;;AAGA,MAAa,YACX,YAC4D;CAC5D,IAAI,OAAO,QAAQ,QAAQ,YAAY,QAAQ,IAAI,KAAK,MAAM,IAC5D,MAAM,IAAI,MAAM,8CAA8C;CAChE,MAAM,SAAS,QAAQ,QAAQ,GAAG;CAClC,IAAI,QAAQ,cAAc,QAAQ,WAAW,MAAM,GACjD,MAAM,IAAI,MAAM,GAAG,OAAO,iDAAiD;CAC7E,OAAO;EAAE;EAAQ,WAAW,GAAG,OAAO,GAAG,WAAW,EAAE;CAAM;AAC9D;;AAGA,MAAM,SAAS,OACb,OACA,SACiF;CACjF,IAAI;CACJ,IAAI;EACF,QAAQ,MAAM,MAAM,IAAI,IAAI;CAC9B,QAAQ;EACN;CACF;CACA,IAAI,UAAU,KAAA,GAAW,OAAO,KAAA;CAChC,MAAM,SAAS,SAAS,KAAK;CAC7B,OAAO,OAAO,SAAS,KAAK,MAAM,OAAO;EAAE;EAAO;CAAO,IAAI,KAAA;AAC/D;AAEA,MAAM,UAAU,gBACd,aAAa,MAAM,GAAG,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,YAAY,KAAK;;AAGtD,MAAM,cAAc,QAAuB,YACzC,OAAO,KACL,GAAG,KAAK,UAAU;CAChB,SAAS;EACP,IAAI,OAAO;EACX,QAAQ,OAAO;EACf,MAAM,OAAO;EACb,GAAI,OAAO,gBAAgB,KAAA,IAAY,EAAE,aAAa,OAAO,YAAY,IAAI,CAAC;EAC9E,GAAI,OAAO,gBAAgB,KAAA,IAAY,EAAE,aAAa,OAAO,YAAY,IAAI,CAAC;EAC9E,aAAa,OAAO;EACpB,cAAc,OAAO;EACrB,SAAS,OAAO;CAClB;CACA,QAAQ;AACV,CAAC,EAAE,KACH,MACF;;;;;;AAOF,MAAa,eAAe,OAC1B,MACA,MACA,UACA,aAC2B;CAC3B,MAAM,SAAS;EACb,UAAU;EACV,WAAW;EACX,UAAU;EACV,aAAa;EACb,UAAU;EACV,UAAU;EACV,SAAS;EACT,SAAS;CACX;CACA,MAAM,EAAE,WAAW;CACnB,MAAM,MAAM,OACV,UACkE;EAClE,MAAM,SAAS,MAAM,UAAU,KAAK;EACpC,MAAM,SAAS,KAAK;EACpB,MAAM,KAAK,MAAM,MAAM;EACvB,OAAO;GAAE;GAAQ,QAAQ,OAAO;EAAO;CACzC;CAEA,MAAM,OAAO,aAAa,WAAW,EAAE;CACvC,MAAM,QAAQ,OAAO,YACnB,OAAO,QAAQ;EACb,OAAO,OAAO;EACd,OAAO,OAAO;EACd,MAAM,OAAO;EACb,YAAY,OAAO;EACnB,KAAK,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,OAAO,IAAI;EACvD,kBAAkB,KAAK;CACzB,CAAC,CAAC,CAAC,QAAQ,GAAG,WAAW,UAAU,KAAA,CAAS,CAC9C;CACA,MAAM,SAAS;EACb,4BAA4B,QAAQ,KAAK,KAAK;EAC9C;EACA;EACA;EACA;EACA,0BAA0B,KAAK,UAAU,KAAK;CAChD;CACA,MAAM,IACJ,YACE;EACE,CAAC,aAAa,UAAU;EACxB,CAAC,kBAAkB,IAAI;EACvB,CAAC,8BAAa,IAAI,KAAK,EAAA,CAAE,YAAY,CAAC;EACtC,CAAC,iBAAiB,QAAQ;EAC1B,CAAC,gBAAgB,yBAAyB;CAC5C,GACA,CAAC,OAAO,KAAK,GAAG,OAAO,KAAK,IAAI,IAAI,QAAQ,MAAM,CAAC,CACrD,CACF;CAGA,MAAM,4BAAY,IAAI,IAGpB;CACF,MAAM,YACJ,SACA,aACS;EACT,UAAU,OAAO,OAAO;EACxB,UAAU,IAAI,SAAS,QAAQ;EAC/B,IAAI,UAAU,OAAO,WAAW,UAAU,OAAO,UAAU,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,KAAM;CACjF;CAEA,MAAM,OAAO,KAAK,SAAS,KACzB;EAAE,QAAQ;EAAS,OAAO,OAAO;CAAM,GACvC,EAAE,iBAAiB,MAAM,OAAO,WAAW,GAAG,CAChD;CACA,WAAW,MAAM,UAAU,MAAM;EAC/B,IAAI,OAAO,WAAW,WAAW,CAAC,MAAM,IAAI,OAAO,IAAI,GAAG;EAC1D,MAAM,QAAQ,QAAQ,MAAM;EAC5B,IAAI,UAAU,KAAA,KAAa,CAAC,SAAS,QAAQ,OAAO,MAAM,GAAG;EAC7D,OAAO,YAAY;EACnB,MAAM,SAA2C;GAC/C,CAAC,aAAa,MAAM,IAAI;GACxB,CAAC,mBAAmB,MAAM,GAAG;GAC7B,CAAC,oBAAoB,IAAI;EAC3B;EACA,MAAM,SAA2C,CAAC,CAAC,+BAA+B,OAAO,EAAE,CAAC;EAC5F,IAAI;EACJ,IAAI,UAAkC;GAAE,QAAQ;GAAQ,QAAQ;EAAU;EAE1E,IAAI,OAAO,SAAS,aAAa,MAAM,SAAS,KAAA,GAAW;GAEzD,MAAM,WAAW,MAAM,cAAc,KAAA,IAAY,KAAA,IAAY,UAAU,IAAI,MAAM,SAAS;GAC1F,MAAM,OAAO,SAAS,KAAK,CACzB,GAAI,MAAM,gBAAgB,KAAA,IAAY,CAAC,CAAC,gBAAgB,MAAM,WAAW,CAAU,IAAI,CAAC,GACxF,GAAG,MAAM,OACX,CAAC;GACD,MAAM,SAAS,SAAS,OAAO,KAAK,MAAM,MAAM,KAAK,CAAC;GACtD,OAAO,SAAS,QAAQ,SAAS;GACjC,MAAM,KAAK,MAAM,IACf,YACE;IACE,CAAC,aAAa,SAAS;IACvB,CAAC,kBAAkB,IAAI;IACvB,GAAG;IACH,CAAC,gBAAgB,mBAAmB;IACpC,GAAI,aAAa,KAAA,IAAY,CAAC,CAAC,kBAAkB,SAAS,EAAE,CAAU,IAAI,CAAC;IAC3E,CAAC,6BAA6B,UAAU,OAAO,MAAM,GAAG;IACxD,GAAI,aAAa,KAAA,IAAY,CAAC,CAAC,uBAAuB,SAAS,IAAI,CAAU,IAAI,CAAC;IAClF,CAAC,uBAAuB,MAAM;IAC9B,CAAC,qBAAqB,SAAS,SAAS,IAAI,CAAC,CAAC;IAC9C,GAAG;IACH,CAAC,+BAA+B,SAAS;IACzC,CAAC,gBAAgB,aAAa;GAChC,GACA,CAAC,IAAI,CACP,CACF;GACA,OAAO,YAAY;GACnB,UAAU;IAAE,QAAQ;IAAW,cAAc;GAAU;GACvD,MAAM,WAAW;IACf,MAAM;IACN,KAAK,MAAM;IACX,MAAM,MAAM;IACZ,QAAQ;IACR,MAAM;IACN;IACA,GAAG;IACH,SAAS,OAAO;IAChB,MAAM;GACR,CAAC;EACH,OAAO,IAAI,OAAO,SAAS,aAAa,MAAM,SAAS,KAAA,GAAW;GAChE,MAAM,OAAO,MAAM,OAAO,KAAK,OAAO,MAAM,IAAI;GAChD,IAAI,SAAS,KAAA,GAAW;IACtB,OAAO,WAAW;IAClB,UAAU;KAAE,QAAQ;KAAQ,QAAQ;IAAe;GACrD,OAAO,IAAI,MAAM,UAAU;IAEzB,MAAM,SAAS,SAAS,KAAK,MAAM;IACnC,OAAO,SAAS,QAAQ,YAAY;IACpC,MAAM,IACJ,YACE;KACE,CAAC,aAAa,YAAY;KAC1B,CAAC,kBAAkB,IAAI;KACvB,GAAG;KACH,CAAC,uBAAuB,MAAM;KAC9B,CAAC,qBAAqB,MAAM;KAC5B,GAAG;KACH,CAAC,gBAAgB,MAAM,eAAe,0BAA0B;IAClE,GACA,CAAC,KAAK,KAAK,CACb,CACF;IACA,OAAO,eAAe;IACtB,UAAU;KAAE,QAAQ;KAAc,MAAM;IAAW;GACrD,OAAO,IAAI,MAAM,WAAW,KAAA,KAAa,MAAM,UAAU,OAAO,MAAM,UAAU,KAAK;IACnF,MAAM,OAAO,SAAS,MAAM,QAAQ;KAClC,GAAI,MAAM,gBAAgB,KAAA,IACtB,CAAC,CAAC,gBAAgB,MAAM,WAAW,CAAU,IAC7C,CAAC;KACL,CAAC,kBAAkB,OAAO,KAAK,MAAM,MAAM,CAAC;KAC5C,GAAG,MAAM;IACX,CAAC;IACD,MAAM,SAAS,SAAS,KAAK,MAAM;IACnC,OAAO,SAAS,QAAQ,UAAU;IAElC,MAAM,YAAY,MAAM,MAAM,eAAe,MAAM,YAAY,WAAW,KAAA;IAC1E,MAAM,KAAK,MAAM,IACf,YACE;KACE,CAAC,aAAa,UAAU;KACxB,CAAC,kBAAkB,IAAI;KACvB,GAAG;KACH,CAAC,uBAAuB,MAAM;KAC9B,CAAC,qBAAqB,SAAS,SAAS,MAAM,KAAK,KAAK,CAAC,CAAC;KAC1D,GAAI,cAAc,KAAA,IAAY,CAAC,CAAC,kBAAkB,SAAS,CAAU,IAAI,CAAC;KAC1E,GAAG;KACH,CAAC,+BAA+B,SAAS;KACzC,CAAC,gBAAgB,aAAa;IAChC,GACA,CAAC,MAAM,KAAK,KAAK,CACnB,CACF;IACA,OAAO,aAAa;IACpB,UAAU;KAAE,QAAQ;KAAY,cAAc;KAAW,MAAM;IAAU;IACzE,SAAS,OAAO,IAAI;KAAE,IAAI;KAAM,MAAM,MAAM;KAAM,KAAK,MAAM;IAAI,CAAC;IAClE,MAAM,WAAW;KACf,MAAM;KACN,KAAK,MAAM;KACX,MAAM,MAAM;KACZ,QAAQ,MAAM;KACd,MAAM,OAAO,MAAM,WAAW;KAC9B;KACA,GAAG;KACH,SAAS,OAAO;KAChB,MAAM,MAAM,WAAW,OAAO,MAAM,WAAW;IACjD,CAAC;GACH;EACF;EAEA,IAAI,SAAS,KAAA,GAAW;GACtB,IAAI,QAAQ,cAAc,WAAW,OAAO,YAAY;GACxD,IAAI,CAAC,KAAK,iBAAiB;EAC7B;EACA,MAAM,QAAQ,WAAW,QAAQ,OAAO;EACxC,MAAM,IACJ,YACE;GACE,CAAC,aAAa,UAAU;GACxB,CAAC,kBAAkB,SAAS,QAAQ,UAAU,CAAC;GAC/C,GAAG;GACH,GAAI,SAAS,KAAA,IAAY,CAAC,CAAC,sBAAsB,IAAI,CAAU,IAAI,CAAC;GACpE,CAAC,qBAAqB,SAAS,SAAS,KAAK,CAAC,CAAC;GAC/C,GAAG;GACH,CAAC,gBAAgB,kBAAkB;EACrC,GACA,CAAC,KAAK,CACR,CACF;EACA,OAAO,YAAY;CACrB;CACA,OAAO;AACT;;;;;;;AAQA,MAAa,QAAQ,UAA0B;CAC7C,IAAI;CACJ,IAAI;EACF,MAAM,IAAI,IAAI,KAAK;CACrB,QAAQ;EACN,OAAO,MAAM,YAAY;CAC3B;CACA,MAAM,OAAO,IAAI,SACd,YAAY,CAAC,CACb,QAAQ,OAAO,EAAE,CAAC,CAClB,QAAQ,aAAa,EAAE;CAC1B,MAAM,MAAM,KAAK,WAAW,GAAG,IAAI,OAAO,KAAK,MAAM,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,GAAG;CAC5E,IAAI,OAAO,IAAI,SAAS,YAAY;CACpC,IAAI,KAAK,SAAS,KAAK,KAAK,SAAS,GAAG,GAAG,OAAO,KAAK,MAAM,GAAG,EAAE;CAClE,MAAM,QAAQ,IAAI,OAAO,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG;CAC1F,OAAO,GAAG,MAAM,IAAI,SAAS,KAAK,KAAK,IAAI,IAAI,OAAO,GAAG,OAAO,UAAU,KAAK,KAAK,IAAI;AAC1F;;AAGA,MAAa,YAAY,QAAwB,aAC/C,GAAG,KAAK,OAAO,GAAG,EAAE,GAAG,OAAO,KAAK,MAAM,GAAG,EAAE,CAAC,CAAC,QAAQ,OAAO,EAAE,EAAE,GAAG,KAAK,UAAU;CACnF,KAAK,OAAO;CACZ,MAAM,OAAO;CACb,QAAQ,OAAO,OAAO,MAAM;CAC5B,QAAQ,OAAO;CACf,QAAQ,OAAO,OAAO,MAAM;CAC5B,QAAQ,OAAO,OAAO,MAAM;CAC5B;AACF,CAAC;;;;;;;;;;AAWH,MAAa,aAAa,OACxB,QACA,YACwB;CACxB,MAAM,OAAO,YAAY,QAAQ,OAAO;CACxC,MAAM,EAAE,QAAQ,cAAc,SAAS,OAAO;CAC9C,MAAM,OAAO,MAAM,YAAY,OAAO,SAAS;CAC/C,IAAI;EACF,MAAM,SAAS,MAAM,aAAa,MAAM,MAAM,SAAS,MAAM,CAAC;EAC9D,MAAM,OAAO,MAAM,KAAK,MAAM;EAC9B,MAAM,OAAO,WAAW,MAAM;EAC9B,OAAO;GAAE,QAAQ;GAAQ,KAAK;GAAQ,OAAO,KAAK;GAAM,GAAG;EAAO;CACpE,SAAS,OAAO;EACd,MAAM,KAAK,QAAQ;EACnB,MAAM,GAAG,WAAW,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;EAC1D,MAAM;CACR;AACF"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package",
   "name": "polite-crawler",
-  "version": "0.2.4",
+  "version": "0.3.0",
   "description": "A polite web crawler for research and AI agents. It fetches a source once, keeps it, and reads it many times: robots.txt obeyed, one request per site, full-text search. The MCP server is the package polite-crawler-mcp.",
   "keywords": [
     "crawler",
@@ -108,7 +108,7 @@
   },
   "devDependencies": {
     "@huggingface/transformers": "^4.3.1",
-    "@mark1russell7/cue": "github:mark1russell7/cue",
+    "@mark1russell7/cue": "github:mark1russell7/cue#f2bb6026e14fcfb782eef3e02771ce983d4bff55",
     "@microsoft/api-extractor": "^7.59.3",
     "@modelcontextprotocol/sdk": "^1.32.1",
     "@types/node": "^22.19.0",
@@ -130,7 +130,8 @@
     "test": "vitest run",
     "test:watch": "vitest",
     "test:coverage": "vitest run --coverage",
-    "typecheck": "tsc -p tsconfig.json --noEmit --emitDeclarationOnly false && tsc -p tsconfig.test.json && tsc -p eval/tsconfig.json"
+    "typecheck": "tsc -p tsconfig.json --noEmit --emitDeclarationOnly false && tsc -p tsconfig.test.json && tsc -p eval/tsconfig.json",
+    "version": "node ../../scripts/version.ts"
   },
   "types": "./build/index.d.mts"
 }