single-file-cli 2.3.1 → 2.4.0

package/lib/version.js

@@ -1 +1 @@
-export const version = "2.3.1";
+export const version = "2.4.0";

package/options.js

@@ -56,6 +56,7 @@ const OPTIONS_INFO = [{
 	"browser-server": { description: "Server to connect to", type: "string", alias: "browser-remote-debugging-url" },
 	"browser-headless": { description: "Run the browser in headless mode", type: "boolean", defaultValue: true },
 	"browser-executable-path": { description: "Path to chrome/chromium executable", type: "string" },
+	"browser-profile": { description: "Path of the browser profile directory to use, e.g. to save pages requiring a logged-in session (see --create-browser-profile). The directory is copied before starting the browser and is left unmodified.", type: "string" },
 	"browser-width": { description: "Width of the browser viewport in pixels", type: "number", defaultValue: 1280 },
 	"browser-height": { description: "Height of the browser viewport in pixels", type: "number", defaultValue: 720 },
 	"browser-debug": { description: "Enable debug mode", type: "boolean" },
@@ -203,6 +204,7 @@ const OPTIONS_INFO = [{
 
 	"save-raw-page": { description: "Save the original page without interpreting it into the browser", type: "boolean" },
 	"output-json": { description: "Output the result as a JSON string containing the page and network info", type: "boolean" },
+	"create-browser-profile": { description: "Path of the browser profile directory to create or update instead of saving a page. The browser is started with a visible window on the URL passed as argument, log in to the website and quit the browser to save the profile, then pass it to --browser-profile when saving pages", type: "string" },
 
 }, {
 	"help": { description: "Show help", type: "boolean" },
@@ -303,12 +305,22 @@ function getOptions() {
 	invalidOptions.forEach(({ name, value }) => errorMessages.push(value === undefined ?
 		`Missing value for --${name}` :
 		`Invalid value for --${name}: ${JSON.stringify(value)}`));
-	if (!urls.length && !options.urlsFile) {
+	if (!urls.length && !options.urlsFile && !options.createBrowserProfile) {
 		errorMessages.push("The URL or path of the page to save is required");
 	}
 	if (urls.length > 2) {
 		errorMessages.push(`Unexpected arguments: ${urls.slice(2).join(", ")}`);
 	}
+	if (options.createBrowserProfile) {
+		if (options.browserProfile) {
+			errorMessages.push("--create-browser-profile cannot be used with --browser-profile, it already takes the path of the profile directory");
+		}
+		if (options.browserServer) {
+			errorMessages.push("--create-browser-profile cannot be used with --browser-server");
+		}
+	} else if (options.browserProfile && options.browserServer) {
+		errorMessages.push("--browser-profile cannot be used with --browser-server");
+	}
 	if (!options.crawlLinks) {
 		const explicitOptions = parseArgs(Array.from(args), false).options;
 		Object.keys(CRAWL_LINKS_DEPENDENT_OPTIONS)

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "2.3.1",
+	"version": "2.4.0",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"license": "AGPL-3.0-or-later",

package/single-file-cli-api.js

@@ -26,7 +26,7 @@
 import { Buffer } from "node:buffer";
 import * as backend from "./lib/cdp-client.js";
 import { getZipScriptSource } from "./lib/single-file-script.js";
-import { createPagesArchive } from "./lib/archive-packager.js";
+import { createPagesArchive } from "./lib/single-file-archive.js";
 import { Deno, path } from "./lib/deno-polyfill.js";
 
 const VALID_URL_TEST = /^(https?|file):\/\//;
@@ -252,7 +252,7 @@ async function runNextTask() {
 		task.status = STATE_PROCESSED;
 		if (options.crawlLinks || tasks.length > 1) {
 			const processedCount = tasks.filter(task => task.status == STATE_PROCESSED).length;
-			const filenameInfo = pageData && pageData.filename && !options.crawlSaveArchive ? " (" + pageData.filename + ")" : "";
+			const filenameInfo = pageData && pageData.filename && !options.crawlSaveArchive && !options.dumpContent ? " (" + pageData.filename + ")" : "";
 			// written to stderr so that stdout stays parseable when using --dump-content
 			console.error(`[${processedCount}/${tasks.length}] ${pageData ? "saved" : "failed"} ${task.url}${filenameInfo}`); // eslint-disable-line no-console
 		}

package/single-file-launcher.js

@@ -22,11 +22,12 @@
  */
 
 import { initialize } from "./single-file-cli-api.js";
-import { closeBrowser } from "./lib/browser.js";
+import { closeBrowser, createBrowserProfile, getBrowserOptions } from "./lib/browser.js";
 import { Deno } from "./lib/deno-polyfill.js";
 import { getOptions, applySettings, parseUrlsFile } from "./options.js";
 
-const { readTextFile, readFile, exit, addSignalListener } = Deno;
+const { readTextFile, readFile, exit, addSignalListener, build } = Deno;
+const QUIT_BROWSER_HINT = build.os == "darwin" ? " (Cmd+Q)" : "";
 
 try {
 	addSignalListener("SIGTERM", closeBrowserAndExit);
@@ -49,6 +50,10 @@ async function run() {
 			const settings = JSON.parse(await readTextFile(options.settingsFile));
 			applySettings(options, settings);
 		}
+		if (options.createBrowserProfile) {
+			await saveBrowserProfile(options);
+			exit(0);
+		}
 		if (options.urlsFile) {
 			urls = await getUrlsFile(options.urlsFile);
 		} else {
@@ -81,6 +86,13 @@ async function run() {
 	}
 }
 
+async function saveBrowserProfile(options) {
+	const profileDirectory = options.createBrowserProfile;
+	console.error(`Log in to the website in the browser window, then quit the browser${QUIT_BROWSER_HINT} to save the profile.`); // eslint-disable-line no-console
+	await createBrowserProfile(Object.assign(getBrowserOptions(options), { profile: profileDirectory, startUrl: options.url }));
+	console.error(`Profile saved, use it with --browser-profile ${JSON.stringify(profileDirectory)}.`); // eslint-disable-line no-console
+}
+
 function parseCookies(textValue) {
 	const httpOnlyRegExp = /^#HttpOnly_(.*)/;
 	return textValue.split(/\r\n|\n/)

package/test/e2e/automation-detection.test.js

@@ -0,0 +1,103 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import process from "node:process";
+
+const execFileAsync = promisify(execFile);
+const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+
+test("pages are not captured as controlled by automation", { timeout: 120000 }, async () => {
+	const server = createServer((_, response) => response
+		.writeHead(200, { "content-type": "text/html" })
+		.end("<html><head><title>Automation</title></head><body><p id=result></p>" +
+			"<script>document.getElementById(\"result\").textContent = \"webdriver=\" + navigator.webdriver;</script>" +
+			"</body></html>"));
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const outputPath = join(directory, "page.html");
+		const url = "http://localhost:" + server.address().port + "/";
+		const { stderr } = await execFileAsync(process.execPath, [
+			"single-file-node.js", url, outputPath
+		], { cwd: cliDirectory });
+		let content;
+		try {
+			content = await readFile(outputPath, "utf8");
+		} catch (error) {
+			throw new Error("missing output file, stderr: " + stderr, { cause: error });
+		}
+		assert.ok(content.includes("webdriver=false"), "expected navigator.webdriver to be false, got: " + content.match(/webdriver=\w+/));
+	} finally {
+		await rm(directory, { recursive: true });
+		server.close();
+	}
+});
+
+test("pages are not captured with a headless user agent", { timeout: 120000 }, async () => {
+	const server = createServer((request, response) => response
+		.writeHead(200, { "content-type": "text/html" })
+		.end("<html><head><title>User agent</title></head><body>" +
+			"<p id=header-agent>header-agent=" + request.headers["user-agent"] + "</p>" +
+			"<p id=script-agent></p>" +
+			"<script>document.getElementById(\"script-agent\").textContent = \"script-agent=\" + navigator.userAgent;</script>" +
+			"</body></html>"));
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const outputPath = join(directory, "page.html");
+		const url = "http://localhost:" + server.address().port + "/";
+		const { stderr } = await execFileAsync(process.execPath, [
+			"single-file-node.js", url, outputPath
+		], { cwd: cliDirectory });
+		let content;
+		try {
+			content = await readFile(outputPath, "utf8");
+		} catch (error) {
+			throw new Error("missing output file, stderr: " + stderr, { cause: error });
+		}
+		const headerAgent = content.match(/header-agent=([^<]*)/);
+		const scriptAgent = content.match(/script-agent=([^<]*)/);
+		assert.ok(headerAgent, "missing the user agent sent to the server");
+		assert.ok(scriptAgent, "missing the user agent read in the page");
+		assert.ok(!headerAgent[1].includes("Headless"), "unexpected headless token in the user agent sent to the server: " + headerAgent[1]);
+		assert.ok(!scriptAgent[1].includes("Headless"), "unexpected headless token in the user agent read in the page: " + scriptAgent[1]);
+		assert.ok(scriptAgent[1].includes("Chrome/"), "expected a Chrome user agent, got: " + scriptAgent[1]);
+	} finally {
+		await rm(directory, { recursive: true });
+		server.close();
+	}
+});
+
+test("the user agent option overrides the browser user agent", { timeout: 120000 }, async () => {
+	const userAgent = "Mozilla/5.0 (SingleFile test) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36";
+	const server = createServer((request, response) => response
+		.writeHead(200, { "content-type": "text/html" })
+		.end("<html><head><title>User agent</title></head><body>" +
+			"<p id=header-agent>header-agent=" + request.headers["user-agent"] + "</p>" +
+			"</body></html>"));
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const outputPath = join(directory, "page.html");
+		const url = "http://localhost:" + server.address().port + "/";
+		const { stderr } = await execFileAsync(process.execPath, [
+			"single-file-node.js", url, outputPath, "--user-agent=" + userAgent
+		], { cwd: cliDirectory });
+		let content;
+		try {
+			content = await readFile(outputPath, "utf8");
+		} catch (error) {
+			throw new Error("missing output file, stderr: " + stderr, { cause: error });
+		}
+		assert.ok(content.includes("header-agent=" + userAgent), "expected the user agent option to be sent, got: " + content.match(/header-agent=[^<]*/));
+	} finally {
+		await rm(directory, { recursive: true });
+		server.close();
+	}
+});

package/test/e2e/browser-profile.test.js

@@ -0,0 +1,67 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, mkdir, readdir, readFile, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import process from "node:process";
+
+const execFileAsync = promisify(execFile);
+const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+
+test("a browser profile is copied and left unmodified by a capture", { timeout: 120000 }, async () => {
+	const server = createServer((_, response) => response
+		.writeHead(200, { "content-type": "text/html" })
+		.end("<html><head><title>Profile</title></head><body>content</body></html>"));
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const profilePath = join(directory, "profile");
+		await mkdir(join(profilePath, "Default"), { recursive: true });
+		await writeFile(join(profilePath, "Default", "Preferences"), "{}");
+		const outputPath = join(directory, "page.html");
+		const url = "http://localhost:" + server.address().port + "/";
+		const { stderr } = await execFileAsync(process.execPath, [
+			"single-file-node.js", url, outputPath,
+			"--browser-profile", profilePath
+		], { cwd: cliDirectory });
+		let content;
+		try {
+			content = await readFile(outputPath, "utf8");
+		} catch (error) {
+			throw new Error("missing output file, stderr: " + stderr, { cause: error });
+		}
+		assert.ok(content.includes("<title>Profile</title>"));
+		assert.deepEqual(await readdir(profilePath), ["Default"]);
+		assert.deepEqual(await readdir(join(profilePath, "Default")), ["Preferences"]);
+		assert.equal(await readFile(join(profilePath, "Default", "Preferences"), "utf8"), "{}");
+	} finally {
+		await rm(directory, { recursive: true });
+		server.close();
+	}
+});
+
+test("a missing browser profile is reported with its path", { timeout: 120000 }, async () => {
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const profilePath = join(directory, "missing");
+		let exitCode = 0;
+		let stderr = "";
+		try {
+			await execFileAsync(process.execPath, [
+				"single-file-node.js", "https://example.com", join(directory, "page.html"),
+				"--browser-profile", profilePath
+			], { cwd: cliDirectory });
+		} catch (error) {
+			exitCode = error.code;
+			stderr = error.stderr;
+		}
+		assert.notEqual(exitCode, 0);
+		assert.ok(stderr.includes(`The browser profile directory was not found at ${JSON.stringify(profilePath)}`));
+	} finally {
+		await rm(directory, { recursive: true });
+	}
+});

package/test/unit/archive-packager.test.js

@@ -1,7 +1,6 @@
 import { test } from "node:test";
 import assert from "node:assert/strict";
-import { createPagesArchive } from "../../lib/archive-packager.js";
-import { configure, ZipWriter, ZipReader, Uint8ArrayReader, Uint8ArrayWriter, TextReader, TextWriter } from "../../lib/single-file-archive.js";
+import { createPagesArchive, configure, ZipWriter, ZipReader, Uint8ArrayReader, Uint8ArrayWriter, TextReader, TextWriter } from "../../lib/single-file-archive.js";
 
 configure({ useWebWorkers: false });
 

package/test/unit/browser-profile.test.js

@@ -0,0 +1,70 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtemp, mkdir, readdir, readFile, symlink, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { copyProfile } from "../../lib/browser.js";
+
+async function createProfileDirectory() {
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	const sourcePath = join(directory, "profile");
+	await mkdir(join(sourcePath, "Default", "Cache"), { recursive: true });
+	await mkdir(join(sourcePath, "Default", "Local Storage"), { recursive: true });
+	await mkdir(join(sourcePath, "GPUCache"), { recursive: true });
+	await writeFile(join(sourcePath, "Local State"), "state");
+	await writeFile(join(sourcePath, "Default", "Cookies"), "cookies");
+	await writeFile(join(sourcePath, "Default", "Cache", "data_0"), "cached");
+	await writeFile(join(sourcePath, "Default", "Local Storage", "leveldb"), "storage");
+	await writeFile(join(sourcePath, "GPUCache", "index"), "cached");
+	await symlink("hostname-1234", join(sourcePath, "SingletonLock"));
+	return { directory, sourcePath, destinationPath: join(directory, "copy") };
+}
+
+test("copying a profile keeps the session data", async () => {
+	const { directory, sourcePath, destinationPath } = await createProfileDirectory();
+	try {
+		await mkdir(destinationPath);
+		await copyProfile(sourcePath, destinationPath);
+		assert.equal(await readFile(join(destinationPath, "Local State"), "utf8"), "state");
+		assert.equal(await readFile(join(destinationPath, "Default", "Cookies"), "utf8"), "cookies");
+		assert.equal(await readFile(join(destinationPath, "Default", "Local Storage", "leveldb"), "utf8"), "storage");
+	} finally {
+		await rm(directory, { recursive: true });
+	}
+});
+
+test("copying a profile skips caches and lock files", async () => {
+	const { directory, sourcePath, destinationPath } = await createProfileDirectory();
+	try {
+		await mkdir(destinationPath);
+		await copyProfile(sourcePath, destinationPath);
+		assert.deepEqual((await readdir(destinationPath)).sort(), ["Default", "Local State"]);
+		assert.deepEqual((await readdir(join(destinationPath, "Default"))).sort(), ["Cookies", "Local Storage"]);
+	} finally {
+		await rm(directory, { recursive: true });
+	}
+});
+
+test("copying a profile leaves the source directory unmodified", async () => {
+	const { directory, sourcePath, destinationPath } = await createProfileDirectory();
+	try {
+		await mkdir(destinationPath);
+		await copyProfile(sourcePath, destinationPath);
+		assert.deepEqual((await readdir(sourcePath)).sort(), ["Default", "GPUCache", "Local State", "SingletonLock"]);
+		assert.deepEqual((await readdir(join(sourcePath, "Default"))).sort(), ["Cache", "Cookies", "Local Storage"]);
+	} finally {
+		await rm(directory, { recursive: true });
+	}
+});
+
+test("copying a missing profile is reported with its path", async () => {
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const missingPath = join(directory, "missing");
+		await assert.rejects(
+			() => copyProfile(missingPath, directory),
+			error => error.message == `The browser profile directory was not found at ${JSON.stringify(missingPath)}`);
+	} finally {
+		await rm(directory, { recursive: true });
+	}
+});

package/test/unit/cli-usage.test.js

@@ -49,6 +49,21 @@ test("a wrong browser executable path is reported with the path", async () => {
 	assert.ok(stderr.includes("The browser executable was not found at \"/nonexistent/chrome\""));
 });
 
+test("creating a browser profile does not require a url", async () => {
+	const { code, stderr } = await runCli(["--create-browser-profile", "/path/to/profile", "--browser-executable-path", "/nonexistent/chrome"]);
+	assert.notEqual(code, 0);
+	assert.equal(stderr.includes("The URL or path of the page to save is required"), false);
+});
+
+test("conflicting browser profile options are reported as errors", async () => {
+	const { code, stderr } = await runCli(["--create-browser-profile", "/path/to/profile", "--browser-profile", "/path/to/other"]);
+	assert.equal(code, 1);
+	assert.ok(stderr.includes("--create-browser-profile cannot be used with --browser-profile"));
+	const remoteResult = await runCli(["https://example.com", "--browser-profile", "/path/to/profile", "--browser-server", "http://localhost:9222"]);
+	assert.equal(remoteResult.code, 1);
+	assert.ok(remoteResult.stderr.includes("--browser-profile cannot be used with --browser-server"));
+});
+
 test("unexpected extra arguments are reported as errors", async () => {
 	const { code, stderr } = await runCli(["https://example.com", "out.html", "extra.html"]);
 	assert.equal(code, 1);

package/lib/archive-packager.js

@@ -1,228 +0,0 @@
-/*
- * Copyright 2010-2026 Gildas Lormeau
- * contact : gildas.lormeau <at> gmail.com
- *
- * This file is part of SingleFile.
- *
- *   The code in this file is free software: you can redistribute it and/or
- *   modify it under the terms of the GNU Affero General Public License
- *   (GNU AGPL) as published by the Free Software Foundation, either version 3
- *   of the License, or (at your option) any later version.
- *
- *   The code in this file is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
- *   General Public License for more details.
- *
- *   As additional permission under GNU AGPL version 3 section 7, you may
- *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU
- *   AGPL normally required by section 4, provided you include this license
- *   notice and a URL through which recipients can access the Corresponding
- *   Source.
- */
-
-/* global URL */
-
-import {
-	configure,
-	createArchive,
-	TextReader,
-	Uint8ArrayReader,
-	Uint8ArrayWriter,
-	ZipReader
-} from "./single-file-archive.js";
-
-const PAGES_PREFIX = "pages/";
-const PAGES_FILENAME = "sfz-pages.json";
-const TOC_FILENAME = "sfz-toc.html";
-const TOC_TITLE = "Table of contents";
-const TOC_STYLE = "body{font-family:system-ui,sans-serif;margin:2em auto;max-width:40em;padding:0 1em;background-color:#fff;color:#000}" +
-	"a{color:#0000ee}a:visited{color:#551a8b}" +
-	"summary{cursor:pointer;font-weight:bold;margin:.5em 0}" +
-	"details{padding-left:1em}ul{margin:.25em 0;padding-left:1.5em}" +
-	"@media(prefers-color-scheme:dark){body{background-color:#111;color:#eee}a{color:#8ab4f8}a:visited{color:#c58af9}}";
-const COMMENT_HEADER = "Page saved with SingleFile";
-const SYMLINK_UNIX_MODE = 0o120777;
-
-export { createPagesArchive };
-
-async function createPagesArchive(pages, options) {
-	configure({ useWebWorkers: false });
-	const manifest = {
-		pages: pages.map((page, pageIndex) => ({
-			path: getPagePath(pageIndex),
-			url: page.url,
-			originalUrls: page.originalUrls,
-			title: page.title
-		}))
-	};
-	if (options.markUnarchivedLinks) {
-		manifest.markUnarchivedLinks = true;
-	}
-	if (options.pageTransitions && options.pageTransitions != "auto") {
-		manifest.pageTransitions = options.pageTransitions;
-	}
-	const pageData = {
-		doctype: "<!DOCTYPE html>",
-		content: "",
-		title: pages[0].title || "",
-		comment: options.insertSingleFileComment ? getComment(pages[0].url, options) : undefined,
-		tocContent: getTOCContent(pages)
-	};
-	const archiveOptions = {
-		url: pages[0].url,
-		multiPageArchive: true,
-		selfExtractingArchive: options.selfExtractingArchive,
-		extractDataFromPage: options.extractDataFromPage,
-		preventAppendedData: options.preventAppendedData,
-		includeBOM: options.includeBOM,
-		insertMetaCSP: options.insertMetaCSP,
-		insertCanonicalLink: options.insertCanonicalLink,
-		insertMetaNoIndex: options.insertMetaNoIndex
-	};
-	const writtenEntries = options.dedupPages ? new Map() : undefined;
-	const aliases = {};
-	const blob = await createArchive(pageData, archiveOptions, options.zipScript, async zipWriter => {
-		for (let pageIndex = 0; pageIndex < pages.length; pageIndex++) {
-			const pagePath = getPagePath(pageIndex);
-			const zipReader = new ZipReader(new Uint8ArrayReader(await pages[pageIndex].getData()));
-			for (const entry of await zipReader.getEntries()) {
-				const filename = pagePath + entry.filename;
-				const rawData = await entry.getData(new Uint8ArrayWriter(), { passThrough: true, checkSignature: false });
-				const canonicalFilename = writtenEntries && findDuplicate(writtenEntries, filename, entry, rawData);
-				if (canonicalFilename === undefined) {
-					await zipWriter.add(filename, new Uint8ArrayReader(rawData), {
-						passThrough: true,
-						compressionMethod: entry.compressionMethod,
-						uncompressedSize: entry.uncompressedSize,
-						signature: entry.signature,
-						comment: entry.comment,
-						lastModDate: entry.lastModDate
-					});
-				} else {
-					// the duplicate becomes a symlink entry so that external
-					// extractors still produce complete page folders, the router
-					// resolves it from the manifest alias map instead
-					aliases[filename] = canonicalFilename;
-					await zipWriter.add(filename, new TextReader(getRelativePath(filename, canonicalFilename)), {
-						msDosCompatible: false,
-						unixMode: SYMLINK_UNIX_MODE,
-						level: 0,
-						comment: entry.comment,
-						lastModDate: entry.lastModDate
-					});
-				}
-			}
-			await zipReader.close();
-		}
-		if (Object.keys(aliases).length) {
-			manifest.aliases = aliases;
-		}
-		if (options.tocPage) {
-			await zipWriter.add(TOC_FILENAME, new TextReader(getTOCPageContent(manifest.pages)));
-		}
-		await zipWriter.add(PAGES_FILENAME, new TextReader(JSON.stringify(manifest, null, 2)));
-	});
-	return new Uint8Array(await blob.arrayBuffer());
-}
-
-function findDuplicate(writtenEntries, filename, entry, rawData) {
-	if (entry.directory || !entry.uncompressedSize) {
-		return;
-	}
-	const key = [entry.compressionMethod, entry.uncompressedSize, entry.signature, rawData.length].join(":");
-	const candidates = writtenEntries.get(key);
-	if (candidates) {
-		const match = candidates.find(candidate => equalData(candidate.rawData, rawData));
-		if (match) {
-			return match.filename;
-		}
-		candidates.push({ filename, rawData });
-	} else {
-		writtenEntries.set(key, [{ filename, rawData }]);
-	}
-}
-
-function equalData(dataLeft, dataRight) {
-	return dataLeft.length == dataRight.length && dataLeft.every((value, index) => value == dataRight[index]);
-}
-
-function getRelativePath(filename, targetFilename) {
-	const baseSegments = filename.split("/").slice(0, -1);
-	const targetSegments = targetFilename.split("/");
-	while (baseSegments.length && targetSegments.length > 1 && baseSegments[0] == targetSegments[0]) {
-		baseSegments.shift();
-		targetSegments.shift();
-	}
-	return "../".repeat(baseSegments.length) + targetSegments.join("/");
-}
-
-function getPagePath(pageIndex) {
-	return pageIndex == 0 ? "" : PAGES_PREFIX + (pageIndex + 1) + "/";
-}
-
-function getComment(url, options) {
-	return "\n " + COMMENT_HEADER +
-		" \n url: " + url +
-		(options.removeSavedDate ? " " : " \n saved date: " + new Date()) + "\n";
-}
-
-function getTOCPageContent(pages) {
-	const origins = new Set(pages.map(page => new URL(page.url).origin));
-	const rootGroup = { groups: new Map(), pages: [] };
-	pages.forEach(page => {
-		const url = new URL(page.url);
-		const segments = url.pathname.split("/").slice(1, -1);
-		if (origins.size > 1) {
-			segments.unshift(url.origin);
-		}
-		let group = rootGroup;
-		segments.forEach(segment => {
-			if (!group.groups.has(segment)) {
-				group.groups.set(segment, { groups: new Map(), pages: [] });
-			}
-			group = group.groups.get(segment);
-		});
-		group.pages.push(page);
-	});
-	const title = pages[0].title ? TOC_TITLE + " - " + pages[0].title : TOC_TITLE;
-	return "<!DOCTYPE html><html lang=\"en\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\">" +
-		"<title>" + escapeUnicodeHTML(title) + "</title><style>" + TOC_STYLE + "</style></head><body><main><h1>" +
-		escapeUnicodeHTML(TOC_TITLE) + "</h1>" + getTOCGroupContent(rootGroup) + "</main></body></html>";
-}
-
-// nested details/summary groups stay collapsible without scripts on purpose,
-// the page must remain usable after a plain unzip
-function getTOCGroupContent(group) {
-	let content = "";
-	if (group.pages.length) {
-		content += "<ul>" + group.pages.map(page =>
-			"<li><a href=\"" + escapeUnicodeHTML(page.path + "index.html") + "\">" + escapeUnicodeHTML(page.title || page.url) + "</a></li>").join("") + "</ul>";
-	}
-	group.groups.forEach((childGroup, segment) => {
-		content += "<details open><summary>" + escapeUnicodeHTML(segment) + "</summary>" + getTOCGroupContent(childGroup) + "</details>";
-	});
-	return content;
-}
-
-// unlike the prelude TOC below, the stored page is a UTF-8 entry: only the
-// markup delimiters need escaping, but crawled titles remain untrusted
-function escapeUnicodeHTML(value) {
-	return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
-}
-
-function getTOCContent(pages) {
-	return "<nav><ul>" +
-		pages.map(page => "<li><a href=\"" + escapeHTML(page.url) + "\">" + escapeHTML(page.title || page.url) + "</a></li>").join("") +
-		"</ul></nav>";
-}
-
-// the prelude declares the windows-1252 charset, non-ASCII characters must be
-// encoded as HTML entities to survive it
-function escapeHTML(value) {
-	return Array.from(value).map(character => {
-		const codePoint = character.codePointAt(0);
-		return codePoint < 32 || codePoint > 126 || character == "&" || character == "<" || character == ">" || character == "\"" ?
-			"&#" + codePoint + ";" : character;
-	}).join("");
-}