single-file-cli 2.8.0 → 2.9.0

package/lib/version.js

@@ -1 +1 @@
-export const version = "2.8.0";
+export const version = "2.9.0";

package/options.js

@@ -24,7 +24,9 @@
 import { version } from "./lib/version.js";
 import { Deno } from "./lib/deno-polyfill.js";
 
-const { args, build, exit } = Deno;
+const { args, build, env, exit } = Deno;
+const BROWSER_ENGINE_ENVIRONMENT_VARIABLE = "SINGLE_FILE_BROWSER_ENGINE";
+const BROWSER_ENGINES = ["chromium", "firefox"];
 
 const USAGE_TEXT = `single-file [url] [output]
 
@@ -56,6 +58,7 @@ const OPTIONS_INFO = [{
 	"browser-server": { description: "Server to connect to", type: "string", alias: "browser-remote-debugging-url" },
 	"browser-headless": { description: "Run the browser in headless mode", type: "boolean", defaultValue: true },
 	"browser-executable-path": { description: "Path to chrome/chromium executable", type: "string" },
+	"browser-engine": { description: "Browser engine to use (chromium, firefox). Firefox is driven through WebDriver BiDi and has limitations: --create-browser-profile is not available, --browser-mobile-emulation only sets the viewport size and scale factor, --emulate-media-feature and the pause of --browser-debug are ignored, the requests of service workers do not get the blocked URL patterns and extra HTTP headers, and navigator.webdriver is true in the page", type: "string", defaultValue: "chromium" },
 	"browser-profile": { description: "Path of the browser profile directory to use, e.g. to save pages requiring a logged-in session (see --create-browser-profile). The directory is copied before starting the browser and is left unmodified.", type: "string" },
 	"browser-width": { description: "Width of the browser viewport in pixels", type: "number", defaultValue: 1280 },
 	"browser-height": { description: "Height of the browser viewport in pixels", type: "number", defaultValue: 720 },
@@ -294,6 +297,15 @@ function applySettings(options, settings, explicitOptions = parseArgs(Array.from
 
 function getOptions() {
 	const { positionals, options, invalidOptions } = parseArgs(Array.from(args));
+	const explicitOptions = parseArgs(Array.from(args), false).options;
+	const environmentBrowserEngine = env.get(BROWSER_ENGINE_ENVIRONMENT_VARIABLE);
+	if (explicitOptions.browserEngine === undefined && environmentBrowserEngine) {
+		if (BROWSER_ENGINES.includes(environmentBrowserEngine)) {
+			options.browserEngine = environmentBrowserEngine;
+		} else {
+			invalidOptions.push({ name: "browser-engine", value: environmentBrowserEngine });
+		}
+	}
 	const unknownOptions = positionals.filter(positional => positional.startsWith("--"));
 	const urls = positionals.filter(positional => !positional.startsWith("--"));
 	if (options.help) {
@@ -316,6 +328,9 @@ function getOptions() {
 		errorMessages.push(`Unexpected arguments: ${urls.slice(2).join(", ")}`);
 	}
 	if (options.createBrowserProfile) {
+		if (options.browserEngine == "firefox") {
+			errorMessages.push("--create-browser-profile is not supported with --browser-engine firefox");
+		}
 		if (options.browserProfile) {
 			errorMessages.push("--create-browser-profile cannot be used with --browser-profile, it already takes the path of the profile directory");
 		}
@@ -326,7 +341,6 @@ function getOptions() {
 		errorMessages.push("--browser-profile cannot be used with --browser-server");
 	}
 	if (!options.crawlLinks) {
-		const explicitOptions = parseArgs(Array.from(args), false).options;
 		Object.keys(CRAWL_LINKS_DEPENDENT_OPTIONS)
 			.filter(optionKey => explicitOptions[optionKey] !== undefined)
 			.forEach(optionKey => errorMessages.push(`${CRAWL_LINKS_DEPENDENT_OPTIONS[optionKey]} requires --crawl-links`));
@@ -483,6 +497,9 @@ function parseArgs(args, setDefaultValues = true) {
 		result.options.browserServer = result.options.browserRemoteDebuggingUrl;
 		delete result.options.browserRemoteDebuggingUrl;
 	}
+	if (result.options.browserEngine !== undefined && !BROWSER_ENGINES.includes(result.options.browserEngine)) {
+		invalidOptions.push({ name: "browser-engine", value: result.options.browserEngine });
+	}
 	if (result.options.filenameReplacedCharacters) {
 		const filenameReplacedCharacters = result.options.filenameReplacedCharacters;
 		result.options.filenameReplacedCharacters = [];

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "2.8.0",
+	"version": "2.9.0",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"license": "AGPL-3.0-or-later",
@@ -17,6 +17,7 @@
 	"scripts": {
 		"test": "node --test --test-concurrency=2 \"test/**/*.test.js\"",
 		"test:dev": "SINGLE_FILE_TARGET=dev node --test --test-concurrency=2 \"test/**/*.test.js\"",
+		"test:firefox": "SINGLE_FILE_BROWSER_ENGINE=firefox node --test --test-concurrency=2 \"test/**/*.test.js\"",
 		"lint": "eslint .",
 		"update-core": "bash update-core.sh"
 	},

package/single-file-cli-api.js

@@ -24,7 +24,8 @@
 /* global URL */
 
 import { Buffer } from "node:buffer";
-import * as backend from "./lib/cdp-client.js";
+import * as cdpBackend from "./lib/cdp-client.js";
+import * as bidiBackend from "./lib/bidi-client.js";
 import { getZipScriptSource } from "./lib/single-file-script.js";
 import { createPagesArchive } from "./lib/single-file-archive.js";
 import { Deno, path } from "./lib/deno-polyfill.js";
@@ -61,9 +62,13 @@ const STATE_PROCESSING = "processing";
 const STATE_PROCESSED = "processed";
 
 const { readTextFile, writeTextFile, readFile, writeFile, stdout, mkdir, makeTempDir, remove, stat, errors } = Deno;
-let tasks = [], maxParallelWorkers, sessionFilename, archiveTempDirectory, errorCount = 0;
+let backend = cdpBackend, tasks = [], maxParallelWorkers, sessionFilename, archiveTempDirectory, errorCount = 0;
 
-export { initialize };
+export { initialize, closeBrowser };
+
+async function closeBrowser() {
+	await backend.closeBrowser();
+}
 
 async function initialize(options) {
 	options = Object.assign({}, DEFAULT_OPTIONS, options);
@@ -101,11 +106,12 @@ async function initialize(options) {
 		archiveTempDirectory = await makeTempDir();
 	}
 	maxParallelWorkers = options.maxParallelWorkers || 8;
+	backend = options.browserEngine == "firefox" ? bidiBackend : cdpBackend;
 	try {
 		await backend.initialize(options);
 	} catch (error) {
 		if (error instanceof errors.NotFound) {
-			let message = "Chromium executable not found. ";
+			let message = (options.browserEngine == "firefox" ? "Firefox" : "Chromium") + " executable not found. ";
 			if (options.browserExecutablePath) {
 				message += "Make sure --browser-executable-path is correct.";
 			} else {
@@ -184,7 +190,7 @@ async function finish(options) {
 			}
 		}
 	}
-	if (!options.browserDebug && !options.browserServer) {
+	if (!options.browserDebug) {
 		await backend.closeBrowser();
 	}
 	return errorCount;

package/single-file-launcher.js

@@ -21,8 +21,8 @@
  *   Source.
  */
 
-import { initialize } from "./single-file-cli-api.js";
-import { closeBrowser, createBrowserProfile, getBrowserOptions } from "./lib/browser.js";
+import { initialize, closeBrowser } from "./single-file-cli-api.js";
+import { createBrowserProfile, getChromiumOptions } from "./lib/chromium.js";
 import { Deno } from "./lib/deno-polyfill.js";
 import { getOptions, applySettings, parseUrlsFile } from "./options.js";
 
@@ -93,7 +93,7 @@ async function run() {
 async function saveBrowserProfile(options) {
 	const profileDirectory = options.createBrowserProfile;
 	console.error(`Log in to the website in the browser window, then quit the browser${QUIT_BROWSER_HINT} to save the profile.`); // eslint-disable-line no-console
-	await createBrowserProfile(Object.assign(getBrowserOptions(options), { profile: profileDirectory, startUrl: options.url }));
+	await createBrowserProfile(Object.assign(getChromiumOptions(options), { profile: profileDirectory, startUrl: options.url }));
 	console.error(`Profile saved, use it with --browser-profile ${JSON.stringify(profileDirectory)}.`); // eslint-disable-line no-console
 }
 

package/test/e2e/automation-detection.test.js

@@ -7,11 +7,11 @@ import { mkdtemp, readFile, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import process from "node:process";
-import { cliDirectory } from "../target.js";
+import { cliDirectory, firefox } from "../target.js";
 
 const execFileAsync = promisify(execFile);
 
-test("pages are not captured as controlled by automation", { timeout: 120000 }, async () => {
+test("pages are not captured as controlled by automation", { timeout: 120000, skip: firefox && "navigator.webdriver is always true under the Firefox remote agent" }, async () => {
 	const server = createServer((_, response) => response
 		.writeHead(200, { "content-type": "text/html" })
 		.end("<html><head><title>Automation</title></head><body><p id=result></p>" +
@@ -66,7 +66,8 @@ test("pages are not captured with a headless user agent", { timeout: 120000 }, a
 		assert.ok(scriptAgent, "missing the user agent read in the page");
 		assert.ok(!headerAgent[1].includes("Headless"), "unexpected headless token in the user agent sent to the server: " + headerAgent[1]);
 		assert.ok(!scriptAgent[1].includes("Headless"), "unexpected headless token in the user agent read in the page: " + scriptAgent[1]);
-		assert.ok(scriptAgent[1].includes("Chrome/"), "expected a Chrome user agent, got: " + scriptAgent[1]);
+		const engineToken = firefox ? "Firefox/" : "Chrome/";
+		assert.ok(scriptAgent[1].includes(engineToken), "expected a user agent naming " + engineToken + ", got: " + scriptAgent[1]);
 	} finally {
 		await rm(directory, { recursive: true });
 		server.close();

package/test/e2e/fidelity.test.js

@@ -24,7 +24,7 @@ import { tmpdir } from "node:os";
 import { join, dirname } from "node:path";
 import { pathToFileURL, fileURLToPath } from "node:url";
 import process from "node:process";
-import { cliDirectory, importLibModule, useDevBuild } from "../target.js";
+import { cliDirectory, importLibModule, useDevBuild, firefox } from "../target.js";
 import { openBrowser } from "../fidelity/browser.js";
 import { startServer } from "../fidelity/server.js";
 const { configure, ZipReader, Uint8ArrayReader, TextWriter } = await importLibModule("single-file-archive.js");
@@ -134,7 +134,7 @@ function getDeclaredFontFamilies(saved) {
 	});
 }
 
-test("a saved page keeps the fonts declared inside a frame it cannot read", options, async () => {
+test("a saved page keeps the fonts declared inside a frame it cannot read", { ...options, skip: skip || (firefox && "the font declared by a sandboxed srcdoc frame is lost on Firefox, see the SingleFile backlog") }, async () => {
 	const { comparison, noise } = await compareSaveWithSource("frame-fonts", []);
 	assertNoWorseThanNoise(comparison, noise);
 });

package/test/e2e/password.test.js

@@ -0,0 +1,47 @@
+/* global TextDecoder */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import process from "node:process";
+import { cliDirectory, importLibModule } from "../target.js";
+
+const { configure, ZipReader, Uint8ArrayReader, TextWriter } = await importLibModule("single-file-archive.js");
+const execFileAsync = promisify(execFile);
+const PASSWORD = "s3cret";
+const MARKER = "protected by a password";
+
+configure({ useWebWorkers: false });
+
+test("a password-protected archive decrypts with the password", { timeout: 120000 }, async () => {
+	const server = createServer((_, response) => response
+		.writeHead(200, { "content-type": "text/html" })
+		.end("<html><head><title>locked</title></head><body><p>" + MARKER + "</p></body></html>"));
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const outputPath = join(directory, "out.html");
+		await execFileAsync(process.execPath, [
+			"single-file-node.js", "http://localhost:" + server.address().port + "/", outputPath,
+			"--compress-content", "--password", PASSWORD
+		], { cwd: cliDirectory });
+		const data = new Uint8Array(await readFile(outputPath));
+		assert.ok(!new TextDecoder().decode(data).includes(MARKER), "the page content is readable without the password");
+		const zipReader = new ZipReader(new Uint8ArrayReader(data), { extractPrependedData: true, extractAppendedData: true, password: PASSWORD });
+		const entries = await zipReader.getEntries();
+		const pageEntry = entries.find(entry => entry.filename.endsWith("index.html"));
+		assert.ok(pageEntry, "the archive has no index.html entry");
+		assert.ok(pageEntry.encrypted, "the page entry is not encrypted");
+		const content = await pageEntry.getData(new TextWriter());
+		assert.ok(content.includes(MARKER), "the page entry does not decrypt to the page");
+		await zipReader.close();
+	} finally {
+		await rm(directory, { recursive: true, force: true });
+		server.close();
+	}
+});

package/test/e2e/proxy-auth.test.js

@@ -0,0 +1,53 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { Buffer } from "node:buffer";
+import process from "node:process";
+import { cliDirectory } from "../target.js";
+
+const execFileAsync = promisify(execFile);
+const USERNAME = "single";
+const PASSWORD = "file";
+// a name the browser cannot resolve itself, so the request can only reach the proxy;
+// a loopback address would be bypassed by both engines
+const PAGE_URL = "http://proxy-auth.test/page.html";
+const MARKER = "served through the proxy";
+const CREDENTIALS = "Basic " + Buffer.from(USERNAME + ":" + PASSWORD).toString("base64");
+
+test("proxy credentials answer the 407 challenge of the proxy", { timeout: 120000 }, async () => {
+	const requests = [];
+	const proxy = createServer((request, response) => {
+		const authorization = request.headers["proxy-authorization"];
+		requests.push({ url: request.url, authorization });
+		if (authorization !== CREDENTIALS) {
+			response.writeHead(407, { "proxy-authenticate": "Basic realm=\"single-file\"", "content-type": "text/plain" }).end("proxy authentication required");
+		} else if (request.url === PAGE_URL) {
+			response.writeHead(200, { "content-type": "text/html" }).end("<html><head><title>proxied</title></head><body><p>" + MARKER + "</p></body></html>");
+		} else {
+			response.writeHead(404).end();
+		}
+	});
+	await new Promise(resolve => proxy.listen(0, "127.0.0.1", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const outputPath = join(directory, "out.html");
+		await execFileAsync(process.execPath, [
+			"single-file-node.js", PAGE_URL, outputPath,
+			"--http-proxy-server", "127.0.0.1:" + proxy.address().port,
+			"--http-proxy-username", USERNAME,
+			"--http-proxy-password", PASSWORD
+		], { cwd: cliDirectory });
+		const content = await readFile(outputPath, "utf8");
+		assert.ok(content.includes(MARKER), "the page was not fetched through the proxy");
+		assert.ok(requests.some(({ authorization }) => !authorization), "the proxy never challenged the browser");
+		assert.ok(requests.some(({ url, authorization }) => url === PAGE_URL && authorization === CREDENTIALS), "the credentials were never sent to the proxy");
+	} finally {
+		await rm(directory, { recursive: true, force: true });
+		proxy.close();
+	}
+});

package/test/e2e/service-worker.test.js

@@ -9,7 +9,7 @@ import { mkdtemp, readFile, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import process from "node:process";
-import { cliDirectory } from "../target.js";
+import { cliDirectory, firefox } from "../target.js";
 
 const execFileAsync = promisify(execFile);
 
@@ -44,7 +44,7 @@ navigator.serviceWorker.register("/sw.js").then(async () => {
 });
 </script></body></html>`;
 
-test("network options reach the requests made by a service worker", { timeout: 120000 }, async () => {
+test("network options reach the requests made by a service worker", { timeout: 120000, skip: firefox && "Firefox reports the requests of a service worker as intercepted but refuses to continue them" }, async () => {
 	const STYLE = "rgb(7,7,7)";
 	const requests = [];
 	const server = createServer((request, response) => {

package/test/fidelity/browser.js

@@ -43,8 +43,8 @@ const SETTLE_EXPRESSION = `Promise.race([
 export { openBrowser, BAND_HEIGHT };
 
 async function openBrowser({ headless = true } = {}) {
-	const { launchBrowser, closeBrowser } = await importLibModule("browser.js");
-	cdpOptions.apiUrl = LOCALHOST + (await launchBrowser({ headless }));
+	const { launchChromium, closeChromium } = await importLibModule("chromium.js");
+	cdpOptions.apiUrl = LOCALHOST + (await launchChromium({ headless }));
 	// A command with no limit waits for its answer for ever, and that is the default. One that never
 	// came back took a whole CI run with it: the suite reported nothing but its own test timeout,
 	// the connection stayed wedged, and every check after it in the file was never reached. With a
@@ -87,7 +87,7 @@ async function openBrowser({ headless = true } = {}) {
 
 	async function close() {
 		await CDP.closeTarget(comparisonTarget.targetId).catch(() => { });
-		await closeBrowser();
+		await closeChromium();
 	}
 }
 

package/test/target.js

@@ -21,6 +21,10 @@ const IGNORED_DIRECTORY_NAMES = ["node_modules", "test", "tmp", "dist", "doc"];
 
 const repositoryDirectory = join(dirname(fileURLToPath(import.meta.url)), "..");
 const useDevBuild = process.env.SINGLE_FILE_TARGET === "dev";
+// SINGLE_FILE_BROWSER_ENGINE=firefox runs the same suite through the BiDi client, the CLI reads it
+// as the default of --browser-engine; the checks that hold for one engine only say so by name
+const browserEngine = process.env.SINGLE_FILE_BROWSER_ENGINE || "chromium";
+const firefox = browserEngine === "firefox";
 const cliDirectory = useDevBuild ? join(repositoryDirectory, ".dev") : repositoryDirectory;
 
 if (useDevBuild) {
@@ -74,4 +78,4 @@ function importLibModule(name) {
 	return import(join(cliDirectory, "lib", name));
 }
 
-export { cliDirectory, repositoryDirectory, useDevBuild, importLibModule };
+export { cliDirectory, repositoryDirectory, useDevBuild, importLibModule, browserEngine, firefox };