single-file-cli 2.4.1 → 2.5.0

package/lib/version.js

@@ -1 +1 @@
-export const version = "2.4.1";
+export const version = "2.5.0";

package/options.js

@@ -135,6 +135,7 @@ const OPTIONS_INFO = [{
 	"create-root-directory": { description: "Create a root directory based on the timestamp", type: "boolean" },
 	"extract-data-from-page": { description: "Extract compressed data from the page instead of fetching the page in order to create universal self-extracting HTML files", type: "boolean", defaultValue: true },
 	"prevent-appended-data": { description: "Prevent appending data after the compressed data when creating self-extracting HTML files", type: "boolean" },
+	"declare-appended-data": { description: "Declare the data appended after the compressed data as the comment of the ZIP archive, for readers rejecting undeclared trailing bytes (e.g. java.util.zip); ZIP tools then print that data when listing the archive", type: "boolean" },
 	"embed-screenshot": { description: "Embed a screenshot of the page as a PNG file in the compressed file (self-extracting HTML or ZIP file). When enabled, the resulting file can be read as a ZIP file or a PNG image.", type: "boolean" },
 	"embed-screenshot-options": { description: "Options passed to the CDP method `Page.captureScreenshot()` given as a JSON string (e.g. { \"captureBeyondViewport\": false })", type: "string" },
 	"embedded-image": { description: "Path to a PNG image to embed in the compressed file.", type: "string" },
@@ -166,6 +167,7 @@ const OPTIONS_INFO = [{
 	"crawl-save-archive-dedup": { description: "Deduplicate identical resources shared between pages when using --crawl-save-archive", type: "boolean" },
 	"crawl-save-archive-mark-unarchived-links": { description: "Mark links to pages not saved in the archive when using --crawl-save-archive", type: "boolean" },
 	"crawl-save-archive-page-transitions": { description: "Page transitions when navigating in the archive saved with --crawl-save-archive. The possible values are \"auto\" (default, i.e. transitions run when pages opt in via CSS), \"fade\" and \"none\"", type: "string", defaultValue: "auto" },
+	"crawl-save-archive-page-list": { description: "Insert the list of the crawled pages in the prelude of the archive when using --crawl-save-archive, so that indexing tools can read it without extracting the archive", type: "boolean" },
 	"crawl-save-archive-toc": { description: "Save a table of contents page into the archive when using --crawl-save-archive", type: "boolean" },
 }, {
 	"browser-script": { description: "Path of a script executed in the page (and all the frames) before it is loaded", type: "string[]" },

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "2.4.1",
+	"version": "2.5.0",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"license": "AGPL-3.0-or-later",

package/single-file-cli-api.js

@@ -79,6 +79,9 @@ async function initialize(options) {
 	if (options.crawlSaveArchiveToc && !options.crawlSaveArchive) {
 		throw new Error("--crawl-save-archive-toc requires --crawl-save-archive");
 	}
+	if (options.crawlSaveArchivePageList && !options.crawlSaveArchive) {
+		throw new Error("--crawl-save-archive-page-list requires --crawl-save-archive");
+	}
 	if (options.crawlSaveArchivePageTransitions !== undefined && !["auto", "fade", "none"].includes(options.crawlSaveArchivePageTransitions)) {
 		throw new Error("--crawl-save-archive-page-transitions must be \"auto\", \"fade\" or \"none\"");
 	}
@@ -199,10 +202,12 @@ async function savePagesArchive(options) {
 			dedupPages: options.crawlSaveArchiveDedup,
 			markUnarchivedLinks: options.crawlSaveArchiveMarkUnarchivedLinks,
 			tocPage: options.crawlSaveArchiveToc,
+			pageList: options.crawlSaveArchivePageList,
 			pageTransitions: options.crawlSaveArchivePageTransitions,
 			selfExtractingArchive: options.selfExtractingArchive,
 			extractDataFromPage: options.extractDataFromPage,
 			preventAppendedData: options.preventAppendedData,
+			declareAppendedData: options.declareAppendedData,
 			includeBOM: options.includeBOM,
 			insertMetaCSP: options.insertMetaCSP,
 			insertCanonicalLink: options.insertCanonicalLink,

package/test/e2e/compress-content.test.js

@@ -0,0 +1,93 @@
+/* global URL, TextDecoder */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import process from "node:process";
+import { configure, ZipReader, Uint8ArrayReader } from "../../lib/single-file-archive.js";
+
+const execFileAsync = promisify(execFile);
+const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+const TEST_TIMEOUT = 120000;
+const END_OF_CENTRAL_DIRECTORY_SIGNATURE = 0x06054b50;
+const END_OF_CENTRAL_DIRECTORY_LENGTH = 22;
+const COMMENT_LENGTH_OFFSET = 20;
+
+const capturePromises = new Map();
+
+test("data is appended after the end of central directory record by default", { timeout: TEST_TIMEOUT }, async () => {
+	const { data, stderr } = await getCaptureResult();
+	const { offset, commentLength } = findEndOfCentralDirectory(data);
+	assert.equal(commentLength, 0, "stderr: " + stderr);
+	assert.ok(data.length > offset + END_OF_CENTRAL_DIRECTORY_LENGTH, "the capture appends no data to declare");
+});
+
+test("--declare-appended-data declares the appended data as the archive comment", { timeout: TEST_TIMEOUT }, async () => {
+	const { data, stderr } = await getCaptureResult(true);
+	const { offset, commentLength } = findEndOfCentralDirectory(data);
+	assert.ok(commentLength > 0, "stderr: " + stderr);
+	assert.equal(offset + END_OF_CENTRAL_DIRECTORY_LENGTH + commentLength, data.length,
+		"the comment must cover every byte after the end of central directory record");
+	const comment = new TextDecoder("windows-1252").decode(data.subarray(data.length - commentLength));
+	assert.ok(comment.startsWith("-->"), "the comment must hold the appended data itself, got " + JSON.stringify(comment.slice(0, 32)));
+});
+
+test("the archive stays readable whether or not the appended data is declared", { timeout: TEST_TIMEOUT }, async () => {
+	for (const declareAppendedData of [false, true]) {
+		const { data } = await getCaptureResult(declareAppendedData);
+		configure({ useWebWorkers: false });
+		const zipReader = new ZipReader(new Uint8ArrayReader(data));
+		const entryNames = (await zipReader.getEntries()).map(entry => entry.filename);
+		assert.ok(entryNames.includes("index.html"), "declareAppendedData: " + declareAppendedData);
+	}
+});
+
+function getCaptureResult(declareAppendedData = false) {
+	if (!capturePromises.has(declareAppendedData)) {
+		capturePromises.set(declareAppendedData, runCapture(declareAppendedData));
+	}
+	return capturePromises.get(declareAppendedData);
+}
+
+async function runCapture(declareAppendedData) {
+	const server = createServer((request, response) => {
+		const { pathname } = new URL(request.url, "http://localhost");
+		if (pathname === "/") {
+			response.writeHead(200, { "content-type": "text/html" })
+				.end("<html><head><title>Compressed Page</title></head><body>content</body></html>");
+		} else {
+			response.writeHead(404).end();
+		}
+	});
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const origin = "http://localhost:" + server.address().port;
+		const { stderr } = await execFileAsync(process.execPath, [
+			"single-file-node.js", origin + "/", join(directory, "page.html"),
+			"--compress-content",
+			...(declareAppendedData ? ["--declare-appended-data"] : [])
+		], { cwd: cliDirectory });
+		const data = new Uint8Array(await readFile(join(directory, "page.html")));
+		return { data, stderr };
+	} finally {
+		await rm(directory, { recursive: true });
+		server.close();
+	}
+}
+
+function findEndOfCentralDirectory(data) {
+	const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+	for (let offset = data.length - END_OF_CENTRAL_DIRECTORY_LENGTH; offset >= 0; offset--) {
+		if (view.getUint32(offset, true) === END_OF_CENTRAL_DIRECTORY_SIGNATURE) {
+			return { offset, commentLength: view.getUint16(offset + COMMENT_LENGTH_OFFSET, true) };
+		}
+	}
+	throw new Error("end of central directory record not found");
+}

package/test/e2e/crawl-save-archive.test.js

@@ -96,6 +96,12 @@ test("--crawl-save-archive-toc requires --crawl-save-archive", { timeout: TEST_T
 		error => error.stderr.includes("--crawl-save-archive-toc requires --crawl-save-archive"));
 });
 
+test("--crawl-save-archive-page-list requires --crawl-save-archive", { timeout: TEST_TIMEOUT }, async () => {
+	await assert.rejects(
+		execFileAsync(process.execPath, ["single-file-node.js", "http://localhost/", "--compress-content", "--crawl-save-archive-page-list"], { cwd: cliDirectory }),
+		error => error.stderr.includes("--crawl-save-archive-page-list requires --crawl-save-archive"));
+});
+
 test("--crawl-save-archive-mark-unarchived-links requires --crawl-save-archive", { timeout: TEST_TIMEOUT }, async () => {
 	await assert.rejects(
 		execFileAsync(process.execPath, ["single-file-node.js", "http://localhost/", "--compress-content", "--crawl-save-archive-mark-unarchived-links"], { cwd: cliDirectory }),