single-file-cli 2.0.18 → 2.0.20

package/README.MD

@@ -80,9 +80,6 @@ Make sure Chrome or a Chromium-based browser is installed in the default folder.
 
     ```sh
     unzip master.zip .
-    ```
-
-    ```sh
     cd single-file-cli-master
     ```
 
@@ -90,9 +87,6 @@ Make sure Chrome or a Chromium-based browser is installed in the default folder.
 
     ```sh
     git clone --depth 1 --recursive https://github.com/gildas-lormeau/single-file-cli.git
-    ```
-
-    ```sh
     cd single-file-cli
     ```
 
@@ -130,13 +124,6 @@ Make sure Chrome or a Chromium-based browser is installed in the default folder.
   single-file https://www.wikipedia.org wikipedia.html
   ```
 
-  - Save https://www.wikipedia.org into `wikipedia.html` in the current folder
-    with Firefox instead of Chrome
-
-  ```sh
-  single-file https://www.wikipedia.org wikipedia.html --back-end=webdriver-gecko
-  ```
-
   - Save a list of URLs stored into `list-urls.txt` in the current folder
 
   ```sh

package/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.0.18",
+  "version": "2.0.20",
   "description": "SingleFile CLI",
   "nodeModulesDir": true,
   "exports": {

package/lib/cdp-client.js

@@ -28,8 +28,8 @@ import { getScriptSource, getHookScriptSource } from "./single-file-script.js";
 import { CDP } from "simple-cdp";
 
 const LOAD_TIMEOUT_ERROR = "ERR_LOAD_TIMEOUT";
-const NETWORK_IDLE_STATE = "networkIdle";
-const NETWORK_STATES = ["networkAlmostIdle", "load", "DOMContentLoaded"];
+const INTERACTIVE_TIME_STATE = "InteractiveTime";
+const NETWORK_STATES = ["InteractiveTime", "networkIdle", "networkAlmostIdle", "load", "DOMContentLoaded"];
 const MINIMIZED_WINDOW_STATE = "minimized";
 const SINGLE_FILE_WORLD_NAME = "singlefile";
 const EMPTY_PAGE_URL = "about:blank";
@@ -62,16 +62,14 @@ async function getPageData(options) {
 		}
 		if (options.httpProxyServer && options.httpProxyUsername) {
 			await Fetch.enable({ handleAuthRequests: true });
-			Fetch.addEventListener("authRequired", async ({ params }) => {
-				await Fetch.continueWithAuth({
-					requestId: params.requestId,
-					authChallengeResponse: {
-						response: "ProvideCredentials",
-						username: options.httpProxyUsername,
-						password: options.httpProxyPassword
-					}
-				});
-			});
+			Fetch.addEventListener("authRequired", ({ params }) => Fetch.continueWithAuth({
+				requestId: params.requestId,
+				authChallengeResponse: {
+					response: "ProvideCredentials",
+					username: options.httpProxyUsername,
+					password: options.httpProxyPassword
+				}
+			}));
 			Fetch.addEventListener("requestPaused", ({ params }) => Fetch.continueRequest({ requestId: params.requestId }));
 		}
 		if (options.httpHeaders && options.httpHeaders.length) {
@@ -88,27 +86,6 @@ async function getPageData(options) {
 		if (options.browserCookies && options.browserCookies.length) {
 			await Network.setCookies({ cookies: options.browserCookies });
 		}
-		let debuggerReady;
-		if (options.browserDebug) {
-			await Debugger.enable();
-			debuggerReady = new Promise(resolve => {
-				const SCRIPT_PARSED_EVENT = "scriptParsed";
-				const RESUMED_EVENT = "resumed";
-				Debugger.addEventListener(SCRIPT_PARSED_EVENT, onScriptParsed);
-
-				async function onScriptParsed() {
-					Debugger.removeEventListener(SCRIPT_PARSED_EVENT, onScriptParsed);
-					Debugger.addEventListener(RESUMED_EVENT, onResumed);
-					await Debugger.pause();
-				}
-
-				async function onResumed() {
-					Debugger.removeEventListener(RESUMED_EVENT, onResumed);
-					await Debugger.disable();
-					resolve();
-				}
-			});
-		}
 		await Page.addScriptToEvaluateOnNewDocument({
 			source: getHookScriptSource(),
 			runImmediately: true
@@ -118,65 +95,12 @@ async function getPageData(options) {
 			runImmediately: true,
 			worldName: SINGLE_FILE_WORLD_NAME
 		});
-		await Runtime.enable();
-		const topFrameId = targetInfo.id;
-		let contextId;
-		const CONTEXT_CREATED_EVENT = "executionContextCreated";
-		Runtime.addEventListener(CONTEXT_CREATED_EVENT, executionContextCreated);
-		await Page.enable();
-		await Page.setLifecycleEventsEnabled({ enabled: true });
-		const pageNavigated = Page.navigate({ url: options.url });
-		const pageReady = new Promise(resolve => {
-			const LIFE_CYCLE_EVENT = "lifecycleEvent";
-			const FRAME_NAVIGATED_EVENT = "frameNavigated";
-			let contentLoaded;
-			Page.addEventListener(LIFE_CYCLE_EVENT, onLifecycleEvent);
-			Page.addEventListener(FRAME_NAVIGATED_EVENT, onFrameNavigated);
-
-			async function onLifecycleEvent({ params }) {
-				const { name, frameId } = params;
-				if (frameId === topFrameId) {
-					if (!contentLoaded && name === "init" || name === "firstPaint") {
-						contentLoaded = true;
-					}
-					if (contentLoaded && name === (options.browserWaitUntil || NETWORK_IDLE_STATE)) {
-						if (options.browserWaitDelay) {
-							setTimeout(() => resolve, options.browserWaitDelay);
-						} else {
-							Page.removeEventListener(LIFE_CYCLE_EVENT, onLifecycleEvent);
-							Page.removeEventListener(FRAME_NAVIGATED_EVENT, onFrameNavigated);
-							await Page.setLifecycleEventsEnabled({ enabled: false });
-							await Page.disable();
-							resolve();
-						}
-					}
-				}
-			}
-
-			async function onFrameNavigated({ params }) {
-				const { frame } = params;
-				if (!frame.parentId) {
-					contextId = undefined;
-				}
-			}
-		});
-		let timeoutReady, timeoutId, resolveTimeoutReady;
-		if (options.browserLoadMaxTime) {
-			timeoutReady = new Promise((resolve, reject) => {
-				resolveTimeoutReady = resolve;
-				timeoutId = setTimeout(() => {
-					const error = new Error("Load timeout");
-					error.code = LOAD_TIMEOUT_ERROR;
-					reject(error);
-				}, options.browserLoadMaxTime);
-			});
-		}
-		await Promise.race([Promise.all([pageNavigated, pageReady, debuggerReady]), timeoutReady]);
-		Runtime.removeEventListener(CONTEXT_CREATED_EVENT, executionContextCreated);
-		await Runtime.disable();
-		if (timeoutId) {
-			clearTimeout(timeoutId);
-			resolveTimeoutReady();
+		const [contextId] = await Promise.all([
+			loadPage({ Page, Runtime }, options),
+			options.browserDebug ? waitForDebuggerReady({ Debugger }) : Promise.resolve()
+		]);
+		if (options.browserWaitDelay) {
+			setTimeout(() => resolve, options.browserWaitDelay);
 		}
 		const { result } = await Runtime.evaluate({
 			expression: `singlefile.getPageData(${JSON.stringify(options)})`,
@@ -184,18 +108,14 @@ async function getPageData(options) {
 			returnByValue: true,
 			contextId
 		});
-		const { value } = result;
+		const { value, subtype, description } = result;
+		if (subtype === "error") {
+			throw new Error(description);
+		}
 		if (options.compressContent) {
 			value.content = new Uint8Array(value.content);
 		}
 		return value;
-
-		async function executionContextCreated({ params }) {
-			const { context } = params;
-			if (contextId === undefined && context.name === SINGLE_FILE_WORLD_NAME && context.auxData && context.auxData.frameId === topFrameId) {
-				contextId = context.id;
-			}
-		}
 	} catch (error) {
 		if (error.code === LOAD_TIMEOUT_ERROR && options.browserWaitUntilFallback && options.browserWaitUntil) {
 			const browserWaitUntil = NETWORK_STATES[(NETWORK_STATES.indexOf(options.browserWaitUntil) + 1)];
@@ -212,6 +132,164 @@ async function getPageData(options) {
 	}
 }
 
+async function loadPage({ Page, Runtime }, options) {
+	await Runtime.enable();
+	await Page.enable();
+	try {
+		const loadTimeoutAbortController = new AbortController();
+		const loadTimeoutAbortSignal = loadTimeoutAbortController.signal;
+		const [contextId] = await Promise.race([
+			Promise.all([getTopFrameContextId({ Page, Runtime }, options), Page.navigate({ url: options.url })]),
+			waitForLoadTimeout(loadTimeoutAbortSignal, options.browserLoadMaxTime)
+		]);
+		if (!loadTimeoutAbortSignal.aborted) {
+			loadTimeoutAbortController.abort();
+		}
+		return contextId;
+	} finally {
+		await Runtime.disable();
+		await Page.disable();
+	}
+}
+
+function getTopFrameContextId({ Page, Runtime }, options) {
+	return new Promise((resolve, reject) => {
+		const FRAME_NAVIGATED_EVENT = "frameNavigated";
+		const CONTEXT_CREATED_EVENT = "executionContextCreated";
+		const CONTEXT_DESTROYED_EVENT = "executionContextDestroyed";
+		const contextIds = [];
+		let frameId, navigationAbortController;
+		Page.addEventListener(FRAME_NAVIGATED_EVENT, onFrameNavigated);
+		Runtime.addEventListener(CONTEXT_CREATED_EVENT, onExecutionContextCreated);
+		Runtime.addEventListener(CONTEXT_DESTROYED_EVENT, onExecutionContextDestroyed);
+
+		async function onFrameNavigated({ params }) {
+			const { frame } = params;
+			if (!frame.parentId) {
+				frameId = frame.id;
+				if (navigationAbortController) {
+					navigationAbortController.abort();
+				}
+				navigationAbortController = new AbortController();
+				await onTopFrameNavigated(navigationAbortController.signal);
+			}
+		}
+
+		async function onTopFrameNavigated(navigationSignal) {
+			await waitForPageReady({ Page }, frameId, options, navigationSignal);
+			if (!navigationSignal.aborted) {
+				Page.removeEventListener(FRAME_NAVIGATED_EVENT, onFrameNavigated);
+				Runtime.removeEventListener(CONTEXT_CREATED_EVENT, onExecutionContextCreated);
+				Runtime.removeEventListener(CONTEXT_DESTROYED_EVENT, onExecutionContextDestroyed);
+				let contextId;
+				if (contextIds.length) {
+					let contextIdIndex = 0;
+					do {
+						if (await testExecutionContext(contextIds[contextIdIndex])) {
+							contextId = contextIds[contextIdIndex];
+						}
+						contextIdIndex++;
+					} while (contextId === undefined && contextIdIndex < contextIds.length);
+				}
+				if (contextId === undefined) {
+					reject(new Error("Execution context not found"));
+				} else {
+					resolve(contextId);
+				}
+			}
+		}
+
+		function onExecutionContextCreated({ params }) {
+			const { context } = params;
+			const { name, origin, auxData } = context;
+			if (frameId !== undefined && origin === "://" && name === SINGLE_FILE_WORLD_NAME && auxData && auxData.frameId === frameId) {
+				contextIds.push(context.id);
+			}
+		}
+
+		function onExecutionContextDestroyed({ params }) {
+			const { executionContextId } = params;
+			const index = contextIds.indexOf(executionContextId);
+			if (index > -1) {
+				contextIds.splice(index, 1);
+			}
+		}
+
+		async function testExecutionContext(contextId) {
+			try {
+				const { result } = await Runtime.evaluate({
+					expression: "singlefile !== undefined",
+					contextId
+				});
+				return result.value === true;
+			} catch (error) {
+				// ignored
+			}
+			return false;
+		}
+	});
+}
+
+function waitForLoadTimeout(abortSignal, maxDelay) {
+	return new Promise((resolve, reject) => {
+		const ABORT_EVENT = "abort";
+		abortSignal.addEventListener(ABORT_EVENT, onAbort);
+		const timeoutId = setTimeout(() => {
+			const error = new Error("Load timeout");
+			error.code = LOAD_TIMEOUT_ERROR;
+			reject(error);
+		}, maxDelay);
+
+		function onAbort() {
+			abortSignal.removeEventListener(ABORT_EVENT, onAbort);
+			clearTimeout(timeoutId);
+			resolve();
+		}
+	});
+}
+
+async function waitForPageReady({ Page }, topFrameId, options, navigationSignal) {
+	await Page.setLifecycleEventsEnabled({ enabled: true });
+	try {
+		await new Promise(resolve => {
+			const LIFE_CYCLE_EVENT = "lifecycleEvent";
+			Page.addEventListener(LIFE_CYCLE_EVENT, onLifecycleEvent);
+
+			function onLifecycleEvent({ params }) {
+				const { name, frameId } = params;
+				if (frameId === topFrameId) {
+					if (name === options.browserWaitUntil || name === INTERACTIVE_TIME_STATE || navigationSignal.aborted) {
+						Page.removeEventListener(LIFE_CYCLE_EVENT, onLifecycleEvent);
+						resolve();
+					}
+				}
+			}
+		});
+	} finally {
+		if (!navigationSignal.aborted) {
+			await Page.setLifecycleEventsEnabled({ enabled: false });
+		}
+	}
+}
+
+async function waitForDebuggerReady({ Debugger }) {
+	await Debugger.enable();
+	try {
+		await Debugger.pause();
+		await new Promise(resolve => {
+			const RESUMED_EVENT = "resumed";
+			Debugger.addEventListener(RESUMED_EVENT, onResumed);
+
+			function onResumed() {
+				Debugger.removeEventListener(RESUMED_EVENT, onResumed);
+				resolve();
+			}
+		});
+	} finally {
+		await Debugger.disable();
+	}
+}
+
 function getBrowserOptions(options) {
 	const browserOptions = {};
 	browserOptions.args = options.browserArgs;

package/lib/version.js

@@ -1 +1 @@
-export const version = "2.0.18";
+export const version = "2.0.20";

package/options.js

@@ -49,7 +49,7 @@ const OPTIONS_INFO = {
 	"browser-height": { description: "Height of the browser viewport in pixels", type: "number", defaultValue: 720 },
 	"browser-load-max-time": { description: "Maximum delay of time to wait for page loading in ms", type: "number", defaultValue: 60000 },
 	"browser-wait-delay": { description: "Time to wait before capturing the page in ms", type: "number" },
-	"browser-wait-until": { description: "When to consider the page is loaded (netWorkIdle, netWorkAlmostIdle, load, domContentLoaded)", type: "string", defaultValue: "networkIdle" },
+	"browser-wait-until": { description: "When to consider the page is loaded (InteractiveTime, networkIdle, networkAlmostIdle, load, domContentLoaded)", type: "string", defaultValue: "networkIdle" },
 	"browser-wait-until-fallback": { description: "Retry with the next value of --browser-wait-until when a timeout error is thrown", type: "boolean", defaultValue: true },
 	"browser-debug": { description: "Enable debug mode", type: "boolean" },
 	"browser-script": { description: "Path of a script executed in the page (and all the frames) before it is loaded", type: "string[]" },

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "2.0.18",
+	"version": "2.0.20",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"engines": {