single-file-cli 2.4.1 → 2.6.0

package/lib/version.js

@@ -1 +1 @@
-export const version = "2.4.1";
+export const version = "2.6.0";

package/options.js

@@ -65,7 +65,7 @@ const OPTIONS_INFO = [{
 	"browser-single-process": { description: "Run the browser as a single process (enabled by default on Windows only, current browsers on other platforms do not support this mode)", type: "boolean", defaultValue: build.os == "windows" },
 	"browser-start-minimized": { description: "Minimize the browser", type: "boolean" },
 	"browser-ignore-insecure-certs": { description: "Ignore HTTPs errors", type: "boolean" },
-	"browser-bypass-CSP": { description: "Bypass the Content Security Policy of the page, needed to save pages enforcing Trusted Types with browsers based on Chromium 150 or older", type: "boolean", defaultValue: false }
+	"browser-bypass-csp": { key: "browserBypassCSP", description: "Bypass the Content Security Policy of the page, needed to save pages enforcing Trusted Types with browsers based on Chromium 150 or older", type: "boolean", defaultValue: false }
 }, {
 	"browser-load-max-time": { description: "Maximum delay of time to wait for loading the page in ms", type: "number", defaultValue: 60000 },
 	"browser-capture-max-time": { description: "Maximum delay of time to wait for capturing the page in ms", type: "number", defaultValue: 60000 },
@@ -102,7 +102,7 @@ const OPTIONS_INFO = [{
 	"block-stylesheets": { description: "Block stylesheets", type: "boolean", defaultValue: false },
 	"block-videos": { description: "Block videos", type: "boolean", defaultValue: true },
 	"block-mixed-content": { description: "Block mixed contents", type: "boolean" },
-	"blocked-URL-pattern": { description: "Regular expression matching URLs to block (e.g. 'annoying-banners\\.com')", type: "string[]" }
+	"blocked-url-pattern": { key: "blockedURLPatterns", description: "Regular expression matching URLs to block (e.g. 'annoying-banners\\.com')", type: "string[]" }
 }, {
 	"load-deferred-images": { description: "Load deferred (a.k.a. lazy-loaded) images", type: "boolean", defaultValue: true },
 	"load-deferred-images-dispatch-scroll-event": { description: "Dispatch 'scroll' event when loading deferred images", type: "boolean" },
@@ -112,8 +112,8 @@ const OPTIONS_INFO = [{
 	"max-resource-size-enabled": { description: "Enable removal of embedded resources exceeding a given size", type: "boolean" },
 	"max-resource-size": { description: "Maximum size of embedded resources in MB (i.e. images, stylesheets, scripts and iframes)", type: "number", defaultValue: 10 }
 }, {
-	"compress-CSS": { description: "Compress CSS stylesheets", type: "boolean" },
-	"compress-HTML": { description: "Compress HTML content", type: "boolean", defaultValue: true },
+	"compress-css": { key: "compressCSS", description: "Compress CSS stylesheets", type: "boolean" },
+	"compress-html": { key: "compressHTML", description: "Compress HTML content", type: "boolean", defaultValue: true },
 	"remove-frames": { description: "Remove frames", type: "boolean" },
 	"remove-hidden-elements": { description: "Remove HTML elements which are not displayed", type: "boolean", defaultValue: true },
 	"removed-elements-selector": { description: "Remove specific HTML elements matching the given CSS selectors (comma separated)", type: "string" },
@@ -129,18 +129,20 @@ const OPTIONS_INFO = [{
 	"group-duplicate-stylesheets": { description: "Group duplicate inline stylesheets into a single stylesheet in order to reduce the size of the page", type: "boolean", defaultValue: false }
 }, {
 	"compress-content": { description: "Create a ZIP file instead of an HTML file", type: "boolean" },
+	"disable-compression": { description: "Store the files of the ZIP file without compressing them", type: "boolean" },
 	"self-extracting-archive": { description: "Create a self-extracting (ZIP) HTML file", type: "boolean", defaultValue: true },
 	"password": { description: "Password of the zip file when using --compress-content or --self-extracting-archive", type: "string" },
 	"insert-text-body": { description: "Insert the text of the page into the self-extracting HTML file", type: "boolean" },
 	"create-root-directory": { description: "Create a root directory based on the timestamp", type: "boolean" },
 	"extract-data-from-page": { description: "Extract compressed data from the page instead of fetching the page in order to create universal self-extracting HTML files", type: "boolean", defaultValue: true },
 	"prevent-appended-data": { description: "Prevent appending data after the compressed data when creating self-extracting HTML files", type: "boolean" },
+	"declare-appended-data": { description: "Declare the data appended after the compressed data as the comment of the ZIP archive, for readers rejecting undeclared trailing bytes (e.g. java.util.zip); ZIP tools then print that data when listing the archive", type: "boolean" },
 	"embed-screenshot": { description: "Embed a screenshot of the page as a PNG file in the compressed file (self-extracting HTML or ZIP file). When enabled, the resulting file can be read as a ZIP file or a PNG image.", type: "boolean" },
 	"embed-screenshot-options": { description: "Options passed to the CDP method `Page.captureScreenshot()` given as a JSON string (e.g. { \"captureBeyondViewport\": false })", type: "string" },
-	"embedded-image": { description: "Path to a PNG image to embed in the compressed file.", type: "string" },
+	"embedded-image": { description: "Path to a PNG image to embed in the compressed file. Unlike --embed-screenshot it is also compatible with --crawl-save-archive, where it becomes the image of the whole archive.", type: "string" },
 	"embed-pdf": { description: "Embed a PDF file in the ZIP or self-extracting file. When enabled, the resulting file can be read as a ZIP file or a PDF file.", type: "boolean" },
 	"embed-pdf-options": { description: "Options passed to the CDP method `Page.printToPDF()` given as a JSON string (e.g. { \"pageRanges\": \"1-1\", \"paperWidth\": 11, \"paperHeight\": 8.5 })", type: "string" },
-	"embedded-pdf": { description: "Path to a PDF file to embed in the compressed file.", type: "string" }
+	"embedded-pdf": { description: "Path to a PDF file to embed in the compressed file. Unlike --embed-pdf it is also compatible with --crawl-save-archive, where it becomes the PDF of the whole archive.", type: "string" }
 }, {
 	"filename-template": { description: "Template used to generate the output filename (see https://github.com/gildas-lormeau/SingleFile/wiki/Template-variables-and-functions)", type: "string", defaultValue: "%if-empty<{page-title}|No title> ({date-locale} {time-locale}).{filename-extension}" },
 	"filename-conflict-action": { description: "Action when the filename is conflicting with existing one on the filesystem. The possible values are \"uniquify\" (default), \"overwrite\" and \"skip\"", type: "string", defaultValue: "uniquify" },
@@ -155,17 +157,18 @@ const OPTIONS_INFO = [{
 	"crawl-inner-links-only": { description: "Crawl pages found via inner links only if they are hosted on the same domain", type: "boolean", defaultValue: true },
 	"crawl-no-parent": { description: "Crawl pages found via inner links only if their URLs are not parent of the URL to crawl", type: "boolean" },
 	"crawl-load-session": { description: "Name of the file of the session to load (previously saved with --crawl-save-session or --crawl-sync-session)", type: "string" },
-	"crawl-remove-URL-fragment": { description: "Remove URL fragments found in links", type: "boolean", defaultValue: true },
+	"crawl-remove-url-fragment": { key: "crawlRemoveURLFragment", description: "Remove URL fragments found in links", type: "boolean", defaultValue: true },
 	"crawl-save-session": { description: "Name of the file where to save the state of the session", type: "string" },
 	"crawl-sync-session": { description: "Name of the file where to load and save the state of the session", type: "string" },
 	"crawl-max-depth": { description: "Max depth when crawling pages found in internal and external links (0: infinite)", type: "number", defaultValue: 1 },
 	"crawl-external-links-max-depth": { description: "Max depth when crawling pages found in external links (0: infinite)", type: "number", defaultValue: 1 },
-	"crawl-replace-URLs": { description: "Replace URLs of saved pages with relative paths of saved pages on the filesystem", type: "boolean" },
+	"crawl-replace-urls": { key: "crawlReplaceURLs", description: "Replace URLs of saved pages with relative paths of saved pages on the filesystem", type: "boolean" },
 	"crawl-rewrite-rule": { description: "Rewrite rule used to rewrite URLs of crawled pages", type: "string[]" },
 	"crawl-save-archive": { description: "Save all the crawled pages into a single (self-extracting) ZIP file, requires --compress-content", type: "boolean" },
 	"crawl-save-archive-dedup": { description: "Deduplicate identical resources shared between pages when using --crawl-save-archive", type: "boolean" },
 	"crawl-save-archive-mark-unarchived-links": { description: "Mark links to pages not saved in the archive when using --crawl-save-archive", type: "boolean" },
 	"crawl-save-archive-page-transitions": { description: "Page transitions when navigating in the archive saved with --crawl-save-archive. The possible values are \"auto\" (default, i.e. transitions run when pages opt in via CSS), \"fade\" and \"none\"", type: "string", defaultValue: "auto" },
+	"crawl-save-archive-page-list": { description: "Insert the list of the crawled pages in the prelude of the archive when using --crawl-save-archive, so that indexing tools can read it without extracting the archive", type: "boolean" },
 	"crawl-save-archive-toc": { description: "Save a table of contents page into the archive when using --crawl-save-archive", type: "boolean" },
 }, {
 	"browser-script": { description: "Path of a script executed in the page (and all the frames) before it is loaded", type: "string[]" },
@@ -183,10 +186,10 @@ const OPTIONS_INFO = [{
 	"infobar-position-right": { description: "Position the infobar at the right of the page", type: "string", defaultValue: "16px" },
 	"infobar-position-left": { description: "Position the infobar at the left of the page", type: "string", defaultValue: "" },
 }, {
-	"include-BOM": { description: "Include the UTF-8 BOM into the HTML page", type: "boolean" },
-	"insert-meta-CSP": { description: "Include a <meta> tag with a CSP to avoid potential requests to internet when viewing a page", type: "boolean", defaultValue: true },
+	"include-bom": { key: "includeBOM", description: "Include the UTF-8 BOM into the HTML page, ignored when the page is compressed unless --extract-data-from-page is disabled and no image is embedded", type: "boolean" },
+	"insert-meta-csp": { key: "insertMetaCSP", description: "Include a <meta> tag with a CSP to avoid potential requests to internet when viewing a page", type: "boolean", defaultValue: true },
 	"remove-saved-date": { description: "Remove saved date metadata in HTML header", type: "boolean" },
-	"save-original-URLs": { description: "Save the original URLS in the embedded contents", type: "boolean" },
+	"save-original-urls": { key: "saveOriginalURLs", description: "Save the original URLS in the embedded contents", type: "boolean" },
 	"insert-single-file-comment": { description: "Insert a comment in the HTML header with the URL of the page", type: "boolean", defaultValue: true },
 	"resolve-links": { description: "Resolve link URLs to absolute URLs", type: "boolean", defaultValue: true }
 }, {
@@ -214,7 +217,7 @@ const OPTIONS_INFO = [{
 const CRAWL_LINKS_DEPENDENT_OPTIONS = {
 	crawlInnerLinksOnly: "--crawl-inner-links-only",
 	crawlNoParent: "--crawl-no-parent",
-	crawlRemoveURLFragment: "--crawl-remove-URL-fragment",
+	crawlRemoveURLFragment: "--crawl-remove-url-fragment",
 	crawlMaxDepth: "--crawl-max-depth",
 	crawlExternalLinksMaxDepth: "--crawl-external-links-max-depth",
 	crawlRewriteRules: "--crawl-rewrite-rule"
@@ -327,6 +330,14 @@ function getOptions() {
 			.filter(optionKey => explicitOptions[optionKey] !== undefined)
 			.forEach(optionKey => errorMessages.push(`${CRAWL_LINKS_DEPENDENT_OPTIONS[optionKey]} requires --crawl-links`));
 	}
+	// the byte order mark is written by the CLI when the page is saved as HTML, and by the
+	// compression processor in the prelude of a self-extracting file — but a universal file
+	// declares a single-byte charset the mark would break, and an embedded image must start
+	// with the PNG signature, so there is nowhere left to write it
+	if (options.includeBOM && options.compressContent &&
+		(!options.selfExtractingArchive || options.extractDataFromPage || options.embeddedImage || options.embedScreenshot)) {
+		console.error("Warning: --include-bom is ignored, the file cannot start with a byte order mark. It requires --self-extracting-archive, --extract-data-from-page=false and no embedded image"); // eslint-disable-line no-console
+	}
 	if (errorMessages.length) {
 		printUsage();
 		errorMessages.forEach(message => console.error(`Error: ${message}`)); // eslint-disable-line no-console
@@ -543,6 +554,11 @@ function parseArgs(args, setDefaultValues = true) {
 
 function getOptionKey(optionKeyName, optionInfo) {
 	if (optionInfo) {
+		// option names are lowercase, the keys they fill are not always: the acronyms of
+		// "insertMetaCSP" and its kind cannot be derived from "insert-meta-csp", so they are declared
+		if (optionInfo.key) {
+			return optionInfo.key;
+		}
 		const optionName = optionInfo.alias || optionKeyName;
 		if (isArray(optionInfo.type)) {
 			return kebabToCamelCase(optionName + "s");

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "2.4.1",
+	"version": "2.6.0",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"license": "AGPL-3.0-or-later",

package/single-file-cli-api.js

@@ -79,6 +79,9 @@ async function initialize(options) {
 	if (options.crawlSaveArchiveToc && !options.crawlSaveArchive) {
 		throw new Error("--crawl-save-archive-toc requires --crawl-save-archive");
 	}
+	if (options.crawlSaveArchivePageList && !options.crawlSaveArchive) {
+		throw new Error("--crawl-save-archive-page-list requires --crawl-save-archive");
+	}
 	if (options.crawlSaveArchivePageTransitions !== undefined && !["auto", "fade", "none"].includes(options.crawlSaveArchivePageTransitions)) {
 		throw new Error("--crawl-save-archive-page-transitions must be \"auto\", \"fade\" or \"none\"");
 	}
@@ -89,9 +92,11 @@ async function initialize(options) {
 		if (!options.compressContent) {
 			throw new Error("--crawl-save-archive requires --compress-content");
 		}
-		if (options.embedPdf || options.embeddedPdf || options.embedScreenshot || options.embeddedImage ||
-			options.outputJson || options.insertTextBody || options.password) {
-			throw new Error("--crawl-save-archive is not compatible with --embed-pdf, --embedded-pdf, --embed-screenshot, --embedded-image, --output-json, --insert-text-body and --password");
+		if (options.embedPdf || options.embedScreenshot) {
+			throw new Error("--crawl-save-archive is not compatible with --embed-pdf and --embed-screenshot: an archive has a single PDF and a single image, and they cannot be rendered from one page among many. Use --embedded-pdf and --embedded-image to provide them");
+		}
+		if (options.outputJson || options.insertTextBody || options.password) {
+			throw new Error("--crawl-save-archive is not compatible with --output-json, --insert-text-body and --password");
 		}
 		archiveTempDirectory = await makeTempDir();
 	}
@@ -199,10 +204,14 @@ async function savePagesArchive(options) {
 			dedupPages: options.crawlSaveArchiveDedup,
 			markUnarchivedLinks: options.crawlSaveArchiveMarkUnarchivedLinks,
 			tocPage: options.crawlSaveArchiveToc,
+			pageList: options.crawlSaveArchivePageList,
 			pageTransitions: options.crawlSaveArchivePageTransitions,
 			selfExtractingArchive: options.selfExtractingArchive,
 			extractDataFromPage: options.extractDataFromPage,
 			preventAppendedData: options.preventAppendedData,
+			declareAppendedData: options.declareAppendedData,
+			embeddedPdf: options.embeddedPdf,
+			embeddedImage: options.embeddedImage,
 			includeBOM: options.includeBOM,
 			insertMetaCSP: options.insertMetaCSP,
 			insertCanonicalLink: options.insertCanonicalLink,
@@ -243,6 +252,10 @@ async function runNextTask() {
 			taskOptions.selfExtractingArchive = false;
 			taskOptions.extractDataFromPage = false;
 			taskOptions.createRootDirectory = false;
+			// the faces belong to the archive, not to the pages it holds: leaving them here would
+			// embed the same PDF and the same image in every page, and copy them once per task
+			taskOptions.embeddedPdf = undefined;
+			taskOptions.embeddedImage = undefined;
 			taskOptions.archiveFilename = archiveTempDirectory + "/" + tasks.indexOf(task) + ".zip";
 		}
 		task.status = STATE_PROCESSING;
@@ -364,6 +377,13 @@ async function capturePage(options) {
 		options.zipScript = getZipScriptSource();
 		const pageData = await backend.getPageData(options);
 		content = pageData.content;
+		// the compressed path emits the BOM inside the archive prologue, where only the writer
+		// can place it; for plain HTML it belongs to whoever saves the file, which is the
+		// extension's download layer there and this function here. The string test keeps the
+		// two apart: compressed content arrives as bytes
+		if (options.includeBOM && typeof content == "string") {
+			content = "\ufeff" + content;
+		}
 		if (options.consoleMessagesFile && pageData.consoleMessages) {
 			await writeTextFile(options.consoleMessagesFile, JSON.stringify(pageData.consoleMessages, null, 2));
 		}

package/single-file-launcher.js

@@ -28,6 +28,8 @@ import { getOptions, applySettings, parseUrlsFile } from "./options.js";
 
 const { readTextFile, readFile, exit, addSignalListener, build } = Deno;
 const QUIT_BROWSER_HINT = build.os == "darwin" ? " (Cmd+Q)" : "";
+const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
+const PDF_SIGNATURE = [0x25, 0x50, 0x44, 0x46, 0x2d];
 
 try {
 	addSignalListener("SIGTERM", closeBrowserAndExit);
@@ -69,9 +71,11 @@ async function run() {
 		}
 		if (options.embeddedImage) {
 			options.embeddedImage = Array.from(await readFile(options.embeddedImage));
+			checkSignature(options.embeddedImage, PNG_SIGNATURE, "--embedded-image", "PNG");
 		}
 		if (options.embeddedPdf) {
 			options.embeddedPdf = Array.from(await readFile(options.embeddedPdf));
+			checkSignature(options.embeddedPdf, PDF_SIGNATURE, "--embedded-pdf", "PDF");
 		}
 		options.retrieveLinks = true;
 		const singlefile = await initialize(options);
@@ -125,4 +129,12 @@ async function closeBrowserAndExit(code) {
 
 async function getUrlsFile(urlsFile) {
 	return parseUrlsFile(await readTextFile(urlsFile));
-}
+}
+
+// the faces are read from the bytes the file starts with: a PDF whose header sits further in
+// is dropped by PDF readers, and a file that is not a PNG produces an image nothing can open
+function checkSignature(data, signature, optionName, formatName) {
+	if (signature.some((byte, index) => data[index] != byte)) {
+		throw new Error(optionName + " must be given a " + formatName + " file, and it must start with the " + formatName + " signature");
+	}
+}

package/test/e2e/blocked-url-pattern.test.js

@@ -0,0 +1,54 @@
+/* global URL */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import process from "node:process";
+
+const execFileAsync = promisify(execFile);
+const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+
+test("a blocked URL is never requested", { timeout: 120000 }, async () => {
+	const BLOCKED_STYLE = "rgb(123,45,67)";
+	const KEPT_STYLE = "rgb(89,89,89)";
+	const requestedPaths = [];
+	const server = createServer((request, response) => {
+		const { pathname } = new URL(request.url, "http://localhost");
+		requestedPaths.push(pathname);
+		if (pathname === "/tracker.css") {
+			response.writeHead(200, { "content-type": "text/css" }).end("h2 { color: " + BLOCKED_STYLE + "; }");
+		} else if (pathname === "/kept.css") {
+			response.writeHead(200, { "content-type": "text/css" }).end("h3 { color: " + KEPT_STYLE + "; }");
+		} else {
+			response.writeHead(200, { "content-type": "text/html" }).end(
+				"<html><head><link rel=\"stylesheet\" href=\"/tracker.css\"><link rel=\"stylesheet\" href=\"/kept.css\"></head>" +
+				"<body><h2>blocked</h2><h3>kept</h3></body></html>");
+		}
+	});
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const outputPath = join(directory, "out.html");
+		const url = "http://localhost:" + server.address().port + "/top.html";
+		await execFileAsync(process.execPath, [
+			"single-file-node.js", url, outputPath,
+			"--blocked-URL-pattern", "tracker"
+		], { cwd: cliDirectory });
+		const content = await readFile(outputPath, "utf8");
+		// dropping the response is not blocking: the server must never be asked
+		// for the resource in the first place
+		assert.ok(!requestedPaths.includes("/tracker.css"), "the blocked URL was requested from the server");
+		assert.ok(!content.includes(BLOCKED_STYLE), "a blocked stylesheet leaked into the page");
+		assert.ok(requestedPaths.includes("/kept.css"), "an unblocked resource was not requested");
+		assert.ok(content.includes(KEPT_STYLE), "an unblocked stylesheet was not inlined");
+	} finally {
+		await rm(directory, { recursive: true });
+		server.close();
+	}
+});

package/test/e2e/compress-content.test.js

@@ -0,0 +1,93 @@
+/* global URL, TextDecoder */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import process from "node:process";
+import { configure, ZipReader, Uint8ArrayReader } from "../../lib/single-file-archive.js";
+
+const execFileAsync = promisify(execFile);
+const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+const TEST_TIMEOUT = 120000;
+const END_OF_CENTRAL_DIRECTORY_SIGNATURE = 0x06054b50;
+const END_OF_CENTRAL_DIRECTORY_LENGTH = 22;
+const COMMENT_LENGTH_OFFSET = 20;
+
+const capturePromises = new Map();
+
+test("data is appended after the end of central directory record by default", { timeout: TEST_TIMEOUT }, async () => {
+	const { data, stderr } = await getCaptureResult();
+	const { offset, commentLength } = findEndOfCentralDirectory(data);
+	assert.equal(commentLength, 0, "stderr: " + stderr);
+	assert.ok(data.length > offset + END_OF_CENTRAL_DIRECTORY_LENGTH, "the capture appends no data to declare");
+});
+
+test("--declare-appended-data declares the appended data as the archive comment", { timeout: TEST_TIMEOUT }, async () => {
+	const { data, stderr } = await getCaptureResult(true);
+	const { offset, commentLength } = findEndOfCentralDirectory(data);
+	assert.ok(commentLength > 0, "stderr: " + stderr);
+	assert.equal(offset + END_OF_CENTRAL_DIRECTORY_LENGTH + commentLength, data.length,
+		"the comment must cover every byte after the end of central directory record");
+	const comment = new TextDecoder("windows-1252").decode(data.subarray(data.length - commentLength));
+	assert.ok(comment.startsWith("-->"), "the comment must hold the appended data itself, got " + JSON.stringify(comment.slice(0, 32)));
+});
+
+test("the archive stays readable whether or not the appended data is declared", { timeout: TEST_TIMEOUT }, async () => {
+	for (const declareAppendedData of [false, true]) {
+		const { data } = await getCaptureResult(declareAppendedData);
+		configure({ useWebWorkers: false });
+		const zipReader = new ZipReader(new Uint8ArrayReader(data));
+		const entryNames = (await zipReader.getEntries()).map(entry => entry.filename);
+		assert.ok(entryNames.includes("index.html"), "declareAppendedData: " + declareAppendedData);
+	}
+});
+
+function getCaptureResult(declareAppendedData = false) {
+	if (!capturePromises.has(declareAppendedData)) {
+		capturePromises.set(declareAppendedData, runCapture(declareAppendedData));
+	}
+	return capturePromises.get(declareAppendedData);
+}
+
+async function runCapture(declareAppendedData) {
+	const server = createServer((request, response) => {
+		const { pathname } = new URL(request.url, "http://localhost");
+		if (pathname === "/") {
+			response.writeHead(200, { "content-type": "text/html" })
+				.end("<html><head><title>Compressed Page</title></head><body>content</body></html>");
+		} else {
+			response.writeHead(404).end();
+		}
+	});
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const origin = "http://localhost:" + server.address().port;
+		const { stderr } = await execFileAsync(process.execPath, [
+			"single-file-node.js", origin + "/", join(directory, "page.html"),
+			"--compress-content",
+			...(declareAppendedData ? ["--declare-appended-data"] : [])
+		], { cwd: cliDirectory });
+		const data = new Uint8Array(await readFile(join(directory, "page.html")));
+		return { data, stderr };
+	} finally {
+		await rm(directory, { recursive: true });
+		server.close();
+	}
+}
+
+function findEndOfCentralDirectory(data) {
+	const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+	for (let offset = data.length - END_OF_CENTRAL_DIRECTORY_LENGTH; offset >= 0; offset--) {
+		if (view.getUint32(offset, true) === END_OF_CENTRAL_DIRECTORY_SIGNATURE) {
+			return { offset, commentLength: view.getUint16(offset + COMMENT_LENGTH_OFFSET, true) };
+		}
+	}
+	throw new Error("end of central directory record not found");
+}

package/test/e2e/crawl-save-archive.test.js

@@ -96,6 +96,12 @@ test("--crawl-save-archive-toc requires --crawl-save-archive", { timeout: TEST_T
 		error => error.stderr.includes("--crawl-save-archive-toc requires --crawl-save-archive"));
 });
 
+test("--crawl-save-archive-page-list requires --crawl-save-archive", { timeout: TEST_TIMEOUT }, async () => {
+	await assert.rejects(
+		execFileAsync(process.execPath, ["single-file-node.js", "http://localhost/", "--compress-content", "--crawl-save-archive-page-list"], { cwd: cliDirectory }),
+		error => error.stderr.includes("--crawl-save-archive-page-list requires --crawl-save-archive"));
+});
+
 test("--crawl-save-archive-mark-unarchived-links requires --crawl-save-archive", { timeout: TEST_TIMEOUT }, async () => {
 	await assert.rejects(
 		execFileAsync(process.execPath, ["single-file-node.js", "http://localhost/", "--compress-content", "--crawl-save-archive-mark-unarchived-links"], { cwd: cliDirectory }),

package/test/e2e/http-header.test.js

@@ -0,0 +1,61 @@
+/* global URL */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import process from "node:process";
+
+const execFileAsync = promisify(execFile);
+const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+
+test("extra HTTP headers do not make cross-origin resources send a preflight", { timeout: 120000 }, async () => {
+	const STYLE = "rgb(11,22,33)";
+	const requests = [];
+	// a server sharing its resources with any origin but not answering preflights,
+	// the shape that loses the resource when the request is preflighted
+	const resourceServer = createServer((request, response) => {
+		const { pathname } = new URL(request.url, "http://127.0.0.1");
+		requests.push({ method: request.method, headers: request.headers });
+		if (request.method === "OPTIONS") {
+			response.writeHead(403).end();
+		} else if (pathname === "/cors.css") {
+			response.writeHead(200, { "content-type": "text/css", "access-control-allow-origin": "*" }).end("h2 { color: " + STYLE + "; }");
+		} else {
+			response.writeHead(404).end();
+		}
+	});
+	await new Promise(resolve => resourceServer.listen(0, "127.0.0.1", resolve));
+	const styleUrl = "http://127.0.0.1:" + resourceServer.address().port + "/cors.css";
+	const server = createServer((request, response) => {
+		response.writeHead(200, { "content-type": "text/html" }).end(
+			"<html><head><link rel=\"stylesheet\" href=\"" + styleUrl + "\"></head><body><h2>styled</h2></body></html>");
+	});
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const outputPath = join(directory, "out.html");
+		const url = "http://localhost:" + server.address().port + "/top.html";
+		await execFileAsync(process.execPath, [
+			"single-file-node.js", url, outputPath,
+			"--http-header", "x-test-header=yes"
+		], { cwd: cliDirectory });
+		const content = await readFile(outputPath, "utf8");
+		assert.ok(!requests.some(({ method }) => method === "OPTIONS"), "the extra HTTP header made the request send a preflight");
+		// the browser is what fetched the resource, the backend fetch sends no
+		// sec-fetch-mode and would hide a preflight failure behind a working save
+		const corsRequest = requests.find(({ headers }) => headers["sec-fetch-mode"] === "cors");
+		assert.ok(corsRequest, "the resource was not fetched by the browser");
+		assert.equal(corsRequest.headers["x-test-header"], "yes", "the extra HTTP header was not sent");
+		assert.ok(content.includes(STYLE), "the cross-origin stylesheet was not inlined");
+	} finally {
+		await rm(directory, { recursive: true });
+		server.close();
+		resourceServer.close();
+	}
+});

package/test/e2e/service-worker.test.js

@@ -0,0 +1,89 @@
+/* global URL */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import process from "node:process";
+
+const execFileAsync = promisify(execFile);
+const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+
+// the worker fetches a marked URL instead of passing the request through, so a
+// request reaching the server proves the service worker is the one that made it
+const SERVICE_WORKER = `self.addEventListener("install", () => self.skipWaiting());
+self.addEventListener("activate", event => event.waitUntil(self.clients.claim()));
+self.addEventListener("fetch", event => {
+	const url = new URL(event.request.url);
+	if (url.pathname.endsWith(".css")) {
+		url.searchParams.set("via", "sw");
+		event.respondWith(fetch(url.href));
+	}
+});`;
+
+const PAGE = `<html><head></head><body><h2>styled</h2><h3>tracked</h3>
+<script>
+navigator.serviceWorker.register("/sw.js").then(async () => {
+	await navigator.serviceWorker.ready;
+	if (!navigator.serviceWorker.controller) {
+		await new Promise(resolve => navigator.serviceWorker.addEventListener("controllerchange", resolve, { once: true }));
+	}
+	for (const href of ["/styled.css", "/tracker.css"]) {
+		await new Promise(resolve => {
+			const link = document.createElement("link");
+			link.rel = "stylesheet";
+			link.href = href;
+			link.onload = link.onerror = resolve;
+			document.head.appendChild(link);
+		});
+	}
+});
+</script></body></html>`;
+
+test("network options reach the requests made by a service worker", { timeout: 120000 }, async () => {
+	const STYLE = "rgb(7,7,7)";
+	const requests = [];
+	const server = createServer((request, response) => {
+		const url = new URL(request.url, "http://localhost");
+		requests.push({
+			path: url.pathname,
+			via: url.searchParams.get("via") || "-",
+			header: request.headers["x-test-header"] || "-"
+		});
+		if (url.pathname === "/sw.js") {
+			response.writeHead(200, { "content-type": "text/javascript" }).end(SERVICE_WORKER);
+		} else if (url.pathname === "/styled.css") {
+			response.writeHead(200, { "content-type": "text/css" }).end("h2 { color: " + STYLE + "; }");
+		} else if (url.pathname === "/tracker.css") {
+			response.writeHead(200, { "content-type": "text/css" }).end("h3 { color: rgb(9,9,9); }");
+		} else {
+			response.writeHead(200, { "content-type": "text/html" }).end(PAGE);
+		}
+	});
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const outputPath = join(directory, "out.html");
+		const url = "http://localhost:" + server.address().port + "/top.html";
+		await execFileAsync(process.execPath, [
+			"single-file-node.js", url, outputPath,
+			"--http-header", "x-test-header=yes",
+			"--blocked-URL-pattern", "tracker",
+			"--browser-wait-delay", "3000"
+		], { cwd: cliDirectory });
+		const content = await readFile(outputPath, "utf8");
+		const workerRequest = requests.find(({ path, via }) => path === "/styled.css" && via === "sw");
+		assert.ok(workerRequest, "the service worker never served the stylesheet, the test proves nothing");
+		assert.equal(workerRequest.header, "yes", "the extra HTTP header was not sent by the service worker");
+		assert.ok(!requests.some(({ path }) => path === "/tracker.css"), "the service worker requested a blocked URL");
+		assert.ok(content.includes(STYLE), "the stylesheet served by the service worker was not inlined");
+	} finally {
+		await rm(directory, { recursive: true });
+		server.close();
+	}
+});

package/test/unit/browser-profile.test.js

@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
 import { mkdtemp, mkdir, readdir, readFile, symlink, writeFile, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { copyProfile } from "../../lib/browser.js";
+import { copyProfile, pruneProfile } from "../../lib/browser.js";
 
 async function createProfileDirectory() {
 	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
@@ -57,6 +57,43 @@ test("copying a profile leaves the source directory unmodified", async () => {
 	}
 });
 
+test("pruning a profile deletes the caches at every depth", async () => {
+	const { directory, sourcePath } = await createProfileDirectory();
+	try {
+		await pruneProfile(sourcePath);
+		assert.deepEqual((await readdir(sourcePath)).sort(), ["Default", "Local State", "SingletonLock"]);
+		assert.deepEqual((await readdir(join(sourcePath, "Default"))).sort(), ["Cookies", "Local Storage"]);
+	} finally {
+		await rm(directory, { recursive: true });
+	}
+});
+
+test("pruning a profile keeps the session data", async () => {
+	const { directory, sourcePath } = await createProfileDirectory();
+	try {
+		await pruneProfile(sourcePath);
+		assert.equal(await readFile(join(sourcePath, "Local State"), "utf8"), "state");
+		assert.equal(await readFile(join(sourcePath, "Default", "Cookies"), "utf8"), "cookies");
+		assert.equal(await readFile(join(sourcePath, "Default", "Local Storage", "leveldb"), "utf8"), "storage");
+	} finally {
+		await rm(directory, { recursive: true });
+	}
+});
+
+test("pruning a profile leaves nothing a copy would have taken", async () => {
+	const { directory, sourcePath, destinationPath } = await createProfileDirectory();
+	try {
+		await mkdir(destinationPath);
+		await copyProfile(sourcePath, destinationPath);
+		await pruneProfile(sourcePath);
+		const prunedEntries = (await readdir(sourcePath, { recursive: true })).sort();
+		const copiedEntries = (await readdir(destinationPath, { recursive: true })).sort();
+		assert.deepEqual(prunedEntries.filter(entry => entry != "SingletonLock"), copiedEntries);
+	} finally {
+		await rm(directory, { recursive: true });
+	}
+});
+
 test("copying a missing profile is reported with its path", async () => {
 	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
 	try {

package/test/unit/cli-usage.test.js

@@ -24,6 +24,25 @@ test("help is displayed with an exit code of 0", async () => {
 	assert.ok(stdout.includes("--browser-executable-path"));
 });
 
+test("every option of the help is named in lowercase", async () => {
+	const { stdout } = await runCli(["--help"]);
+	const optionNames = Array.from(stdout.matchAll(/^\s+--(\S+):/gm)).map(([, name]) => name);
+	assert.ok(optionNames.length > 100);
+	assert.deepEqual(optionNames.filter(name => name != name.toLowerCase()), []);
+});
+
+test("a byte order mark that cannot be written is reported as a warning", async () => {
+	const { stderr } = await runCli(["--include-bom", "--compress-content"]);
+	assert.ok(stderr.includes("Warning: --include-bom is ignored"));
+});
+
+test("a byte order mark that can be written is not reported", async () => {
+	const { stderr } = await runCli(["--include-bom", "--compress-content", "--extract-data-from-page=false"]);
+	assert.equal(stderr.includes("--include-bom is ignored"), false);
+	const plain = await runCli(["--include-bom"]);
+	assert.equal(plain.stderr.includes("--include-bom is ignored"), false);
+});
+
 test("a missing url is reported as an error", async () => {
 	const { code, stderr } = await runCli(["--browser-executable-path", "/path/to/chrome"]);
 	assert.equal(code, 1);