single-file-cli 2.1.0 → 2.1.1

package/compile.sh

@@ -9,7 +9,6 @@ trap 'rm -rf "$stage_dir"' EXIT
 cp ./single-file ./single-file-launcher.js ./single-file-cli-api.js ./options.js ./deno.json "$stage_dir"
 cp -R ./lib "$stage_dir/lib"
 cp -R ./resources "$stage_dir/resources"
-jq 'del(.dependencies, .devDependencies)' ./package.json > "$stage_dir/package.json"
 mkdir -p "$dist_dir"
 cd "$stage_dir"
 

package/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.1.0",
+  "version": "2.1.1",
   "description": "SingleFile CLI",
   "exports": {
     ".": "./single-file-cli-api.js"

package/lib/browser.js

@@ -21,7 +21,7 @@
  *   Source.
  */
 
-/* global fetch */
+/* global fetch, setTimeout */
 
 import { BROWSER_PATHS, BROWSER_ARGS } from "./constants.js";
 import { Deno } from "./deno-polyfill.js";
@@ -29,10 +29,12 @@ import { Deno } from "./deno-polyfill.js";
 const NULL_STD_CONFIG = "null";
 const DEBUG_PORT_MIN = 9222;
 const DEBUG_PORT_RANGE = 256;
+const READY_TIMEOUT = 30000;
+const READY_RETRY_DELAY = 250;
 
 const { build, makeTempDir, Command, errors, remove } = Deno;
-let child, profilePath;
-export { launchBrowser, closeBrowser };
+let child, profilePath, childExited;
+export { launchBrowser, closeBrowser, browserExited };
 
 async function launchBrowser(options = {}, indexPath = 0) {
 	const executablePath = options.executablePath || BROWSER_PATHS[build.os][indexPath];
@@ -84,9 +86,47 @@ async function launchBrowser(options = {}, indexPath = 0) {
 		throw error;
 	}
 	child.ref();
+	childExited = false;
+	child.status.then(() => childExited = true);
+	try {
+		await waitUntilReady(debugPort);
+	} catch (error) {
+		await closeBrowser();
+		if (options.singleProcess) {
+			console.warn("Warning: the browser exited when using --browser-single-process, retrying without it"); // eslint-disable-line no-console
+			return launchBrowser(Object.assign({}, options, { singleProcess: false }), indexPath);
+		}
+		throw error;
+	}
 	return debugPort;
 }
 
+async function waitUntilReady(debugPort) {
+	const timeoutTime = Date.now() + READY_TIMEOUT;
+	while (!childExited && Date.now() < timeoutTime) {
+		try {
+			await fetch("http://localhost:" + debugPort + "/json/version");
+			return;
+		} catch {
+			await new Promise(resolve => setTimeout(resolve, READY_RETRY_DELAY));
+		}
+	}
+	throw new Error(childExited ? "The browser exited unexpectedly" : "The browser is not responding");
+}
+
+async function browserExited(maxDelay = 0) {
+	if (child === undefined) {
+		return false;
+	}
+	if (childExited || !maxDelay) {
+		return childExited;
+	}
+	return Promise.race([
+		child.status.then(() => true),
+		new Promise(resolve => setTimeout(() => resolve(childExited), maxDelay))
+	]);
+}
+
 async function getDebugPort(port = getRandomDebugPort(), usedPorts = []) {
 	try {
 		await fetch("http://localhost:" + port + "/json/version");
@@ -111,7 +151,11 @@ function getRandomDebugPort() {
 
 async function closeBrowser() {
 	if (child !== undefined) {
-		child.kill();
+		try {
+			child.kill();
+		} catch {
+			// ignored
+		}
 		await child.status;
 		child = undefined;
 	}

package/lib/cdp-client-util.js

@@ -23,6 +23,7 @@
 
 /* global setTimeout, clearTimeout, fetch, URL, Headers */
 
+import { Buffer } from "node:buffer";
 import { Deno, isDeno, path } from "./deno-polyfill.js";
 
 const ABORT_EVENT = "abort";
@@ -82,7 +83,7 @@ function waitForTimeout(abortSignal, maxDelay, errorMessage, errorCode) {
 }
 
 function arrayBufferToBase64(arrayBuffer) {
-	return new Uint8Array(arrayBuffer).toBase64();
+	return Buffer.from(arrayBuffer).toString("base64");
 }
 
 function getAlternativeUrl(url) {

package/lib/cdp-client.js

@@ -25,7 +25,8 @@
 
 import {
 	launchBrowser,
-	closeBrowser
+	closeBrowser,
+	browserExited
 } from "./browser.js";
 import {
 	CDP,
@@ -57,6 +58,10 @@ const SET_SCREENSHOT_FUNCTION_NAME = "setScreenshot";
 const SET_PDF_FUNCTION_NAME = "setPDF";
 const SET_PAGE_DATA_FUNCTION_NAME = "setPageData";
 const BINDING_CALLED_EVENT_TYPE = "bindingCalled";
+const LOCALHOST = "http://localhost:";
+const BROWSER_EXITED_MAX_DELAY = 2000;
+
+let browserOptions, relaunchBrowserPromise;
 
 export {
 	initialize,
@@ -68,8 +73,7 @@ async function initialize(singleFileOptions) {
 	if (singleFileOptions.browserServer) {
 		options.apiUrl = singleFileOptions.browserServer;
 	} else {
-		const LOCALHOST = "http://localhost:";
-		const browserOptions = {};
+		browserOptions = {};
 		browserOptions.args = singleFileOptions.browserArgs;
 		browserOptions.headless = singleFileOptions.browserHeadless;
 		browserOptions.executablePath = singleFileOptions.browserExecutablePath;
@@ -84,6 +88,18 @@ async function initialize(singleFileOptions) {
 	}
 }
 
+function relaunchBrowser() {
+	if (!relaunchBrowserPromise) {
+		console.warn("Warning: the browser exited when using --browser-single-process, retrying without it"); // eslint-disable-line no-console
+		relaunchBrowserPromise = (async () => {
+			await closeBrowser();
+			browserOptions.singleProcess = false;
+			options.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
+		})();
+	}
+	return relaunchBrowserPromise;
+}
+
 async function getPageData(options) {
 	const EMPTY_PAGE_URL = "about:blank";
 	const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {} };
@@ -108,6 +124,9 @@ async function getPageData(options) {
 		if (shouldRetryWithFallback(error)) {
 			return await retryWithFallback();
 		}
+		if (await shouldRelaunchBrowser()) {
+			return await relaunchBrowserAndRetry();
+		}
 		attachDebugInfo(error, pageContext);
 		throw error;
 	} finally {
@@ -131,9 +150,28 @@ async function getPageData(options) {
 		return await getPageData(options);
 	}
 
+	async function shouldRelaunchBrowser() {
+		return Boolean(browserOptions && browserOptions.singleProcess) && await browserExited(BROWSER_EXITED_MAX_DELAY);
+	}
+
+	async function relaunchBrowserAndRetry() {
+		logData(["Relaunching the browser"], pageContext);
+		targetInfo = null;
+		if (cdp) {
+			cdp.reset();
+			cdp = null;
+		}
+		await relaunchBrowser();
+		return await getPageData(options);
+	}
+
 	async function closeTarget() {
 		if (targetInfo && !options.browserDebug) {
-			await CDP.closeTarget(targetInfo.id);
+			try {
+				await CDP.closeTarget(targetInfo.id);
+			} catch {
+				// ignored
+			}
 			targetInfo = null;
 		}
 		// the connection is closed even when the target is left open for

package/lib/version.js

@@ -1 +1 @@
-export const version = "2.1.0";
+export const version = "2.1.1";

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "2.1.0",
+	"version": "2.1.1",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"engines": {

package/single-file-cli-api.js

@@ -23,6 +23,7 @@
 
 /* global URL */
 
+import { Buffer } from "node:buffer";
 import * as backend from "./lib/cdp-client.js";
 import { getZipScriptSource } from "./lib/single-file-script.js";
 import { Deno, path } from "./lib/deno-polyfill.js";
@@ -292,7 +293,7 @@ async function capturePage(options) {
 		if (options.outputJson) {
 			if (content instanceof Uint8Array) {
 				pageData.content = undefined;
-				pageData.binaryContent = content.toBase64();
+				pageData.binaryContent = Buffer.from(content).toString("base64");
 			}
 			pageData.doctype = undefined;
 			pageData.viewport = undefined;

package/test/e2e/crawl.test.js

@@ -18,8 +18,8 @@ const TEST_TIMEOUT = 120000;
 let crawlPromise;
 
 test("a page linked with and without fragment is captured once", { timeout: TEST_TIMEOUT }, async () => {
-	const { filenames } = await getCrawlResult();
-	assert.ok(filenames.includes("Top Page.html"));
+	const { filenames, stderr } = await getCrawlResult();
+	assert.ok(filenames.includes("Top Page.html"), "missing top page, captured: " + filenames.join(", ") + ", stderr: " + stderr);
 	assert.equal(filenames.filter(filename => filename.startsWith("Linked Page")).length, 1);
 });
 
@@ -66,7 +66,9 @@ test("mail and script links are not crawled as external links", { timeout: TEST_
 			"--errors-file", join(directory, "errors.txt"),
 			"--max-parallel-workers", "1"
 		], { cwd: cliDirectory });
-		assert.deepEqual(await readdir(directory), ["Mail Top.html"]);
+		const filenames = await readdir(directory);
+		const errors = filenames.includes("errors.txt") ? await readFile(join(directory, "errors.txt"), "utf8") : "";
+		assert.deepEqual(filenames, ["Mail Top.html"], "errors: " + errors);
 	} finally {
 		await rm(directory, { recursive: true });
 		server.close();
@@ -115,7 +117,7 @@ async function runCrawl() {
 	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
 	try {
 		const url = "http://localhost:" + server.address().port + "/";
-		await execFileAsync(process.execPath, [
+		const { stderr } = await execFileAsync(process.execPath, [
 			"single-file-node.js", url,
 			"--crawl-links",
 			"--crawl-replace-URLs",
@@ -128,7 +130,7 @@ async function runCrawl() {
 		for (const filename of filenames) {
 			pages[filename] = await readFile(join(directory, filename), "utf8");
 		}
-		return { filenames, pages, otherServerRequests };
+		return { filenames, pages, otherServerRequests, stderr };
 	} finally {
 		await rm(directory, { recursive: true });
 		server.close();

package/test/e2e/frame-gate.test.js

@@ -9,13 +9,14 @@ import { mkdtemp, readFile, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
+import { Buffer } from "node:buffer";
 import process from "node:process";
 
 const execFileAsync = promisify(execFile);
 const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
 const LOAD_MARKER = "MARKER_ADDED_AFTER_LOAD_EVENT";
 const SLOW_RESOURCE_DELAY = 3500;
-const PIXEL_PNG = Uint8Array.fromBase64("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==");
+const PIXEL_PNG = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", "base64");
 
 const TOP_PAGE = `<html><head><title>top</title></head><body>
 <img src="/slow.png">
@@ -29,7 +30,7 @@ addEventListener("load", () => {
 </script>
 </body></html>`;
 
-test("capture waits for the top frame, not a fast iframe", { timeout: 60000 }, async () => {
+test("capture waits for the top frame, not a fast iframe", { timeout: 120000 }, async () => {
 	const server = createServer((request, response) => {
 		const { pathname } = new URL(request.url, "http://localhost");
 		if (pathname === "/top.html") {

package/test/e2e/output-json.test.js

@@ -1,5 +1,3 @@
-/* global TextDecoder */
-
 import { test } from "node:test";
 import assert from "node:assert/strict";
 import { createServer } from "node:http";
@@ -9,13 +7,14 @@ import { mkdtemp, readFile, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
+import { Buffer } from "node:buffer";
 import process from "node:process";
 
 const execFileAsync = promisify(execFile);
 const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
 const ZIP_SIGNATURE = "PK\u0003\u0004";
 
-test("output-json embeds compressed content as base64", { timeout: 60000 }, async () => {
+test("output-json embeds compressed content as base64", { timeout: 120000 }, async () => {
 	const server = createServer((_, response) => response
 		.writeHead(200, { "content-type": "text/html" })
 		.end("<html><head><title>JSON Output</title></head><body>content</body></html>"));
@@ -24,14 +23,20 @@ test("output-json embeds compressed content as base64", { timeout: 60000 }, asyn
 	try {
 		const outputPath = join(directory, "page.json");
 		const url = "http://localhost:" + server.address().port + "/";
-		await execFileAsync(process.execPath, [
+		const { stderr } = await execFileAsync(process.execPath, [
 			"single-file-node.js", url, outputPath,
 			"--compress-content",
 			"--output-json"
 		], { cwd: cliDirectory });
-		const pageData = JSON.parse(await readFile(outputPath, "utf8"));
+		let rawPageData;
+		try {
+			rawPageData = await readFile(outputPath, "utf8");
+		} catch (error) {
+			throw new Error("missing output file, stderr: " + stderr, { cause: error });
+		}
+		const pageData = JSON.parse(rawPageData);
 		assert.ok(pageData.binaryContent);
-		const content = new TextDecoder("latin1").decode(Uint8Array.fromBase64(pageData.binaryContent));
+		const content = Buffer.from(pageData.binaryContent, "base64").toString("latin1");
 		assert.ok(content.includes(ZIP_SIGNATURE));
 	} finally {
 		await rm(directory, { recursive: true });