single-file-cli 2.2.2 → 2.3.1

package/lib/version.js

@@ -1 +1 @@
-export const version = "2.2.2";
+export const version = "2.3.1";

package/options.js

@@ -53,7 +53,7 @@ const CATEGORIES = [
 ];
 
 const OPTIONS_INFO = [{
-	"browser-server": { description: "Server to connect to", type: "string" },
+	"browser-server": { description: "Server to connect to", type: "string", alias: "browser-remote-debugging-url" },
 	"browser-headless": { description: "Run the browser in headless mode", type: "boolean", defaultValue: true },
 	"browser-executable-path": { description: "Path to chrome/chromium executable", type: "string" },
 	"browser-width": { description: "Width of the browser viewport in pixels", type: "number", defaultValue: 1280 },
@@ -64,7 +64,7 @@ const OPTIONS_INFO = [{
 	"browser-single-process": { description: "Run the browser as a single process (enabled by default on Windows only, current browsers on other platforms do not support this mode)", type: "boolean", defaultValue: build.os == "windows" },
 	"browser-start-minimized": { description: "Minimize the browser", type: "boolean" },
 	"browser-ignore-insecure-certs": { description: "Ignore HTTPs errors", type: "boolean" },
-	"browser-remote-debugging-URL": { description: "Remote debugging URL", type: "string" }
+	"browser-bypass-CSP": { description: "Bypass the Content Security Policy of the page, needed to save pages enforcing Trusted Types with browsers based on Chromium 150 or older", type: "boolean", defaultValue: false }
 }, {
 	"browser-load-max-time": { description: "Maximum delay of time to wait for loading the page in ms", type: "number", defaultValue: 60000 },
 	"browser-capture-max-time": { description: "Maximum delay of time to wait for capturing the page in ms", type: "number", defaultValue: 60000 },
@@ -164,6 +164,7 @@ const OPTIONS_INFO = [{
 	"crawl-save-archive": { description: "Save all the crawled pages into a single (self-extracting) ZIP file, requires --compress-content", type: "boolean" },
 	"crawl-save-archive-dedup": { description: "Deduplicate identical resources shared between pages when using --crawl-save-archive", type: "boolean" },
 	"crawl-save-archive-mark-unarchived-links": { description: "Mark links to pages not saved in the archive when using --crawl-save-archive", type: "boolean" },
+	"crawl-save-archive-page-transitions": { description: "Page transitions when navigating in the archive saved with --crawl-save-archive. The possible values are \"auto\" (default, i.e. transitions run when pages opt in via CSS), \"fade\" and \"none\"", type: "string", defaultValue: "auto" },
 	"crawl-save-archive-toc": { description: "Save a table of contents page into the archive when using --crawl-save-archive", type: "boolean" },
 }, {
 	"browser-script": { description: "Path of a script executed in the page (and all the frames) before it is loaded", type: "string[]" },
@@ -335,7 +336,7 @@ function printUsage() {
 				optionType = optionType.replace("[]", "*");
 			}
 			const optionDescription = optionInfo.description;
-			const optionDefaultValue = optionInfo.defaultValue === undefined ? "" : `(default: ${JSON.stringify(optionInfo.defaultValue)})`;
+			const optionDefaultValue = optionInfo.defaultValue === undefined ? "" : `(default: ${JSON.stringify(optionInfo.defaultValue).replace(/[\u007f-\u009f]/g, character => "\\u" + character.charCodeAt(0).toString(16).padStart(4, "0"))})`;
 			console.log(`    --${optionName}: ${optionDescription} <${optionType}> ${optionDefaultValue}`); // eslint-disable-line no-console
 		});
 		console.log(""); // eslint-disable-line no-console
@@ -454,6 +455,10 @@ function parseArgs(args, setDefaultValues = true) {
 		result.options.errorsTracesDisabled = result.options.errorTracesDisabled;
 		delete result.options.errorTracesDisabled;
 	}
+	if (result.options.browserRemoteDebuggingUrl !== undefined) {
+		result.options.browserServer = result.options.browserRemoteDebuggingUrl;
+		delete result.options.browserRemoteDebuggingUrl;
+	}
 	if (result.options.filenameReplacedCharacters) {
 		const filenameReplacedCharacters = result.options.filenameReplacedCharacters;
 		result.options.filenameReplacedCharacters = [];
@@ -556,7 +561,7 @@ function getOptionInfo(optionName) {
 	let result;
 	OPTIONS_INFO.forEach(categoryOptions => {
 		Object.keys(categoryOptions).forEach(keyName => {
-			if (keyName.toLowerCase() == optionName.toLowerCase() || categoryOptions[keyName].alias == optionName.toLowerCase()) {
+			if (keyName.toLowerCase() == optionName.toLowerCase() || (categoryOptions[keyName].alias && categoryOptions[keyName].alias.toLowerCase() == optionName.toLowerCase())) {
 				result = { name: keyName, info: categoryOptions[keyName] };
 			}
 		});

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "2.2.2",
+	"version": "2.3.1",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"license": "AGPL-3.0-or-later",

package/single-file-cli-api.js

@@ -61,7 +61,7 @@ const STATE_PROCESSING = "processing";
 const STATE_PROCESSED = "processed";
 
 const { readTextFile, writeTextFile, readFile, writeFile, stdout, mkdir, makeTempDir, remove, stat, errors } = Deno;
-let tasks = [], maxParallelWorkers, sessionFilename, archiveTempDirectory;
+let tasks = [], maxParallelWorkers, sessionFilename, archiveTempDirectory, errorCount = 0;
 
 export { initialize };
 
@@ -79,6 +79,12 @@ async function initialize(options) {
 	if (options.crawlSaveArchiveToc && !options.crawlSaveArchive) {
 		throw new Error("--crawl-save-archive-toc requires --crawl-save-archive");
 	}
+	if (options.crawlSaveArchivePageTransitions !== undefined && !["auto", "fade", "none"].includes(options.crawlSaveArchivePageTransitions)) {
+		throw new Error("--crawl-save-archive-page-transitions must be \"auto\", \"fade\" or \"none\"");
+	}
+	if (options.crawlSaveArchivePageTransitions !== undefined && options.crawlSaveArchivePageTransitions != "auto" && !options.crawlSaveArchive) {
+		throw new Error("--crawl-save-archive-page-transitions requires --crawl-save-archive");
+	}
 	if (options.crawlSaveArchive) {
 		if (!options.compressContent) {
 			throw new Error("--crawl-save-archive requires --compress-content");
@@ -174,8 +180,9 @@ async function finish(options) {
 		}
 	}
 	if (!options.browserDebug && !options.browserServer) {
-		return backend.closeBrowser();
+		await backend.closeBrowser();
 	}
+	return errorCount;
 }
 
 async function savePagesArchive(options) {
@@ -192,6 +199,7 @@ async function savePagesArchive(options) {
 			dedupPages: options.crawlSaveArchiveDedup,
 			markUnarchivedLinks: options.crawlSaveArchiveMarkUnarchivedLinks,
 			tocPage: options.crawlSaveArchiveToc,
+			pageTransitions: options.crawlSaveArchivePageTransitions,
 			selfExtractingArchive: options.selfExtractingArchive,
 			extractDataFromPage: options.extractDataFromPage,
 			preventAppendedData: options.preventAppendedData,
@@ -242,6 +250,12 @@ async function runNextTask() {
 		task.promise = capturePage(taskOptions);
 		const pageData = await task.promise;
 		task.status = STATE_PROCESSED;
+		if (options.crawlLinks || tasks.length > 1) {
+			const processedCount = tasks.filter(task => task.status == STATE_PROCESSED).length;
+			const filenameInfo = pageData && pageData.filename && !options.crawlSaveArchive ? " (" + pageData.filename + ")" : "";
+			// written to stderr so that stdout stays parseable when using --dump-content
+			console.error(`[${processedCount}/${tasks.length}] ${pageData ? "saved" : "failed"} ${task.url}${filenameInfo}`); // eslint-disable-line no-console
+		}
 		if (pageData) {
 			task.filename = pageData.filename;
 			task.title = pageData.title;
@@ -391,6 +405,7 @@ async function capturePage(options) {
 		}
 		return pageData;
 	} catch (error) {
+		errorCount++;
 		const date = new Date();
 		let message = `[${date.toISOString()}] URL: ${options.url} Error: ${error.message || error}`;
 		if (!options.errorsTracesDisabled) {

package/single-file-launcher.js

@@ -71,7 +71,10 @@ async function run() {
 		options.retrieveLinks = true;
 		const singlefile = await initialize(options);
 		await singlefile.capture(urls);
-		await singlefile.finish();
+		const errorCount = await singlefile.finish();
+		if (errorCount) {
+			exit(1);
+		}
 	} catch (error) {
 		console.error(error.message || error); // eslint-disable-line no-console
 		await closeBrowserAndExit(-1);

package/test/e2e/errors-file.test.js

@@ -11,15 +11,21 @@ import process from "node:process";
 const execFileAsync = promisify(execFile);
 const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
 
-test("errors file lines include the error message", { timeout: 120000 }, async () => {
+test("errors file lines include the error message and the exit code is nonzero", { timeout: 120000 }, async () => {
 	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
 	try {
 		const errorsPath = join(directory, "errors.txt");
 		const url = "http://localhost:1/";
-		await execFileAsync(process.execPath, [
-			"single-file-node.js", url, join(directory, "out.html"),
-			"--errors-file", errorsPath
-		], { cwd: cliDirectory });
+		let exitCode = 0;
+		try {
+			await execFileAsync(process.execPath, [
+				"single-file-node.js", url, join(directory, "out.html"),
+				"--errors-file", errorsPath
+			], { cwd: cliDirectory });
+		} catch (error) {
+			exitCode = error.code;
+		}
+		assert.equal(exitCode, 1);
 		const content = await readFile(errorsPath, "utf8");
 		assert.ok(content.includes("URL: " + url));
 		const errorMessage = content.match(/Error: (.*)/);

package/test/e2e/oopif.test.js

@@ -0,0 +1,86 @@
+/* global URL */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import process from "node:process";
+
+const execFileAsync = promisify(execFile);
+const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+const FRAME_MARKER = "OUT_OF_PROCESS_FRAME_CONTENT";
+
+test("blocked URL patterns and extra HTTP headers apply in cross-origin frames", { timeout: 120000 }, async () => {
+	const BLOCKED_STYLE = "rgb(123,45,67)";
+	const GATED_STYLE = "rgb(89,89,89)";
+	const frameServer = createServer((request, response) => {
+		const { pathname } = new URL(request.url, "http://127.0.0.1");
+		if (pathname === "/tracker.css") {
+			response.writeHead(200, { "content-type": "text/css" }).end("h2 { color: " + BLOCKED_STYLE + "; }");
+		} else if (pathname === "/gated.css") {
+			if (request.headers["x-test-header"] === "yes") {
+				response.writeHead(200, { "content-type": "text/css" }).end("h3 { color: " + GATED_STYLE + "; }");
+			} else {
+				response.writeHead(404).end();
+			}
+		} else {
+			response.writeHead(200, { "content-type": "text/html" }).end(
+				"<html><head><link rel=\"stylesheet\" href=\"/tracker.css\"><link rel=\"stylesheet\" href=\"/gated.css\"></head>" +
+				"<body><h2>blocked</h2><h3>gated</h3></body></html>");
+		}
+	});
+	await new Promise(resolve => frameServer.listen(0, "127.0.0.1", resolve));
+	const frameUrl = "http://127.0.0.1:" + frameServer.address().port + "/frame.html";
+	const server = createServer((request, response) => {
+		response.writeHead(200, { "content-type": "text/html" }).end("<html><body><iframe src=\"" + frameUrl + "\"></iframe></body></html>");
+	});
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const outputPath = join(directory, "out.html");
+		const url = "http://localhost:" + server.address().port + "/top.html";
+		await execFileAsync(process.execPath, [
+			"single-file-node.js", url, outputPath,
+			"--blocked-URL-pattern", "tracker",
+			"--http-header", "x-test-header=yes"
+		], { cwd: cliDirectory });
+		const content = await readFile(outputPath, "utf8");
+		assert.ok(!content.includes(BLOCKED_STYLE), "a blocked stylesheet leaked into the frame");
+		assert.ok(content.includes(GATED_STYLE), "the extra HTTP header was not sent for a frame resource");
+	} finally {
+		await rm(directory, { recursive: true });
+		server.close();
+		frameServer.close();
+	}
+});
+
+test("cross-origin frames are captured in the saved page", { timeout: 120000 }, async () => {
+	// localhost and 127.0.0.1 are different sites, so site isolation renders
+	// the frame out of process, in a separate CDP target
+	const frameServer = createServer((request, response) => {
+		response.writeHead(200, { "content-type": "text/html" }).end("<html><body><p>" + FRAME_MARKER + "</p></body></html>");
+	});
+	await new Promise(resolve => frameServer.listen(0, "127.0.0.1", resolve));
+	const frameUrl = "http://127.0.0.1:" + frameServer.address().port + "/frame.html";
+	const server = createServer((request, response) => {
+		response.writeHead(200, { "content-type": "text/html" }).end("<html><body><iframe src=\"" + frameUrl + "\"></iframe></body></html>");
+	});
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const outputPath = join(directory, "out.html");
+		const url = "http://localhost:" + server.address().port + "/top.html";
+		await execFileAsync(process.execPath, ["single-file-node.js", url, outputPath], { cwd: cliDirectory });
+		const content = await readFile(outputPath, "utf8");
+		assert.ok(content.includes(FRAME_MARKER), "the out-of-process frame was saved empty");
+	} finally {
+		await rm(directory, { recursive: true });
+		server.close();
+		frameServer.close();
+	}
+});

package/test/e2e/urls-file.test.js

@@ -35,3 +35,34 @@ test("duplicate urls in a urls file are captured once", { timeout: 120000 }, asy
 		server.close();
 	}
 });
+
+test("a failed capture in a urls file batch sets a nonzero exit code without aborting the batch", { timeout: 120000 }, async () => {
+	const server = createServer((_, response) => response
+		.writeHead(200, { "content-type": "text/html" })
+		.end("<html><head><title>Good Page</title></head><body>content</body></html>"));
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const goodUrl = "http://localhost:" + server.address().port + "/";
+		const urlsFilePath = join(directory, "urls.txt");
+		await writeFile(urlsFilePath, "http://localhost:1/\n" + goodUrl + "\n");
+		let exitCode = 0, stderr = "";
+		try {
+			({ stderr } = await execFileAsync(process.execPath, [
+				"single-file-node.js",
+				"--urls-file", urlsFilePath,
+				"--output-directory", directory,
+				"--filename-template", "{page-title}.html"
+			], { cwd: cliDirectory }));
+		} catch (error) {
+			exitCode = error.code;
+			stderr = error.stderr;
+		}
+		assert.equal(exitCode, 1);
+		const files = (await readdir(directory)).filter(file => file.endsWith(".html"));
+		assert.deepEqual(files, ["Good Page.html"], "stderr: " + stderr);
+	} finally {
+		await rm(directory, { recursive: true });
+		server.close();
+	}
+});

package/test/unit/archive-packager.test.js

@@ -31,13 +31,26 @@ test("the table of contents groups by origin when the crawl crossed hosts", asyn
 	assert.ok(toc.includes("<summary>https://other.example</summary>"));
 });
 
-async function createArchive(pages) {
+test("the page transitions setting is stored in the manifest except the default", async () => {
+	for (const pageTransitions of [undefined, "auto"]) {
+		const archive = await createArchive([{ url: "https://example.com/", title: "Home" }], { pageTransitions });
+		const manifest = JSON.parse(await readEntry(archive, "sfz-pages.json"));
+		assert.equal(manifest.pageTransitions, undefined, "the default is not stored");
+	}
+	for (const pageTransitions of ["fade", "none"]) {
+		const archive = await createArchive([{ url: "https://example.com/", title: "Home" }], { pageTransitions });
+		const manifest = JSON.parse(await readEntry(archive, "sfz-pages.json"));
+		assert.equal(manifest.pageTransitions, pageTransitions);
+	}
+});
+
+async function createArchive(pages, options = {}) {
 	return createPagesArchive(pages.map(page => ({
 		url: page.url,
 		originalUrls: [page.url],
 		title: page.title,
 		getData: () => createPageData(page)
-	})), { tocPage: true, selfExtractingArchive: false });
+	})), Object.assign({ tocPage: true, selfExtractingArchive: false }, options));
 }
 
 async function createPageData(page) {

package/test/unit/options.test.js

@@ -27,6 +27,9 @@ test("aliases map to the canonical option", () => {
 	assert.equal(parse(["--error-traces-disabled", "false"]).errorsTracesDisabled, false);
 	assert.equal(parse(["--errors-traces-disabled", "false"]).errorsTracesDisabled, false);
 	assert.equal(parse([]).errorsTracesDisabled, true);
+	assert.equal(parse(["--browser-remote-debugging-URL", "http://localhost:9222"]).browserServer, "http://localhost:9222");
+	assert.equal(parse(["--browser-server", "http://localhost:9222"]).browserServer, "http://localhost:9222");
+	assert.equal("browserRemoteDebuggingUrl" in parse(["--browser-remote-debugging-url", "http://localhost:9222"]), false);
 });
 
 test("browser arguments are merged", () => {