single-file-cli 2.0.17 → 2.0.19

package/README.MD

@@ -130,13 +130,6 @@ Make sure Chrome or a Chromium-based browser is installed in the default folder.
   single-file https://www.wikipedia.org wikipedia.html
   ```
 
-  - Save https://www.wikipedia.org into `wikipedia.html` in the current folder
-    with Firefox instead of Chrome
-
-  ```sh
-  single-file https://www.wikipedia.org wikipedia.html --back-end=webdriver-gecko
-  ```
-
   - Save a list of URLs stored into `list-urls.txt` in the current folder
 
   ```sh

package/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.0.17",
+  "version": "2.0.19",
   "description": "SingleFile CLI",
   "nodeModulesDir": true,
   "exports": {

package/lib/cdp-client.js

@@ -28,8 +28,8 @@ import { getScriptSource, getHookScriptSource } from "./single-file-script.js";
 import { CDP } from "simple-cdp";
 
 const LOAD_TIMEOUT_ERROR = "ERR_LOAD_TIMEOUT";
-const NETWORK_IDLE_STATE = "networkIdle";
-const NETWORK_STATES = ["networkAlmostIdle", "load", "DOMContentLoaded"];
+const INTERACTIVE_TIME_STATE = "InteractiveTime";
+const NETWORK_STATES = ["InteractiveTime", "networkIdle", "networkAlmostIdle", "load", "DOMContentLoaded"];
 const MINIMIZED_WINDOW_STATE = "minimized";
 const SINGLE_FILE_WORLD_NAME = "singlefile";
 const EMPTY_PAGE_URL = "about:blank";
@@ -62,16 +62,14 @@ async function getPageData(options) {
 		}
 		if (options.httpProxyServer && options.httpProxyUsername) {
 			await Fetch.enable({ handleAuthRequests: true });
-			Fetch.addEventListener("authRequired", async ({ params }) => {
-				await Fetch.continueWithAuth({
-					requestId: params.requestId,
-					authChallengeResponse: {
-						response: "ProvideCredentials",
-						username: options.httpProxyUsername,
-						password: options.httpProxyPassword
-					}
-				});
-			});
+			Fetch.addEventListener("authRequired", ({ params }) => Fetch.continueWithAuth({
+				requestId: params.requestId,
+				authChallengeResponse: {
+					response: "ProvideCredentials",
+					username: options.httpProxyUsername,
+					password: options.httpProxyPassword
+				}
+			}));
 			Fetch.addEventListener("requestPaused", ({ params }) => Fetch.continueRequest({ requestId: params.requestId }));
 		}
 		if (options.httpHeaders && options.httpHeaders.length) {
@@ -88,27 +86,6 @@ async function getPageData(options) {
 		if (options.browserCookies && options.browserCookies.length) {
 			await Network.setCookies({ cookies: options.browserCookies });
 		}
-		let debuggerReady;
-		if (options.browserDebug) {
-			await Debugger.enable();
-			debuggerReady = new Promise(resolve => {
-				const SCRIPT_PARSED_EVENT = "scriptParsed";
-				const RESUMED_EVENT = "resumed";
-				Debugger.addEventListener(SCRIPT_PARSED_EVENT, onScriptParsed);
-
-				async function onScriptParsed() {
-					Debugger.removeEventListener(SCRIPT_PARSED_EVENT, onScriptParsed);
-					Debugger.addEventListener(RESUMED_EVENT, onResumed);
-					await Debugger.pause();
-				}
-
-				async function onResumed() {
-					Debugger.removeEventListener(RESUMED_EVENT, onResumed);
-					await Debugger.disable();
-					resolve();
-				}
-			});
-		}
 		await Page.addScriptToEvaluateOnNewDocument({
 			source: getHookScriptSource(),
 			runImmediately: true
@@ -118,66 +95,10 @@ async function getPageData(options) {
 			runImmediately: true,
 			worldName: SINGLE_FILE_WORLD_NAME
 		});
-		await Runtime.enable();
-		const topFrameId = targetInfo.id;
-		let contextId;
-		const CONTEXT_CREATED_EVENT = "executionContextCreated";
-		Runtime.addEventListener(CONTEXT_CREATED_EVENT, executionContextCreated);
-		await Page.enable();
-		await Page.setLifecycleEventsEnabled({ enabled: true });
-		const pageNavigated = Page.navigate({ url: options.url });
-		const pageReady = new Promise(resolve => {
-			const LIFE_CYCLE_EVENT = "lifecycleEvent";
-			const FRAME_NAVIGATED_EVENT = "frameNavigated";
-			let contentLoaded;
-			Page.addEventListener(LIFE_CYCLE_EVENT, onLifecycleEvent);
-			Page.addEventListener(FRAME_NAVIGATED_EVENT, onFrameNavigated);
-
-			async function onLifecycleEvent({ params }) {
-				const { name, frameId } = params;
-				if (frameId === topFrameId) {
-					if (name === "DOMContentLoaded") {
-						contentLoaded = true;
-					}
-					if (contentLoaded && name === (options.browserWaitUntil || NETWORK_IDLE_STATE)) {
-						if (options.browserWaitDelay) {
-							setTimeout(() => resolve, options.browserWaitDelay);
-						} else {
-							Page.removeEventListener(LIFE_CYCLE_EVENT, onLifecycleEvent);
-							Page.removeEventListener(FRAME_NAVIGATED_EVENT, onFrameNavigated);
-							await Page.setLifecycleEventsEnabled({ enabled: false });
-							await Page.disable();
-							resolve();
-						}
-					}
-				}
-			}
-
-			async function onFrameNavigated({ params }) {
-				const { frame } = params;
-				if (!frame.parentId) {
-					contextId = undefined;
-				}
-			}
-		});
-		let timeoutReady, timeoutId, resolveTimeoutReady;
-		if (options.browserLoadMaxTime) {
-			timeoutReady = new Promise((resolve, reject) => {
-				resolveTimeoutReady = resolve;
-				timeoutId = setTimeout(() => {
-					const error = new Error("Load timeout");
-					error.code = LOAD_TIMEOUT_ERROR;
-					reject(error);
-				}, options.browserLoadMaxTime);
-			});
-		}
-		await Promise.race([Promise.all([pageNavigated, pageReady, debuggerReady]), timeoutReady]);
-		Runtime.removeEventListener(CONTEXT_CREATED_EVENT, executionContextCreated);
-		await Runtime.disable();
-		if (timeoutId) {
-			clearTimeout(timeoutId);
-			resolveTimeoutReady();
-		}
+		const [contextId] = await Promise.all([
+			navigate({ Page, Runtime }, options),
+			options.browserDebug ? waitForDebuggerReady({ Debugger }) : Promise.resolve()
+		]);
 		const { result } = await Runtime.evaluate({
 			expression: `singlefile.getPageData(${JSON.stringify(options)})`,
 			awaitPromise: true,
@@ -189,13 +110,6 @@ async function getPageData(options) {
 			value.content = new Uint8Array(value.content);
 		}
 		return value;
-
-		async function executionContextCreated({ params }) {
-			const { context } = params;
-			if (contextId === undefined && context.name === SINGLE_FILE_WORLD_NAME && context.auxData && context.auxData.frameId === topFrameId) {
-				contextId = context.id;
-			}
-		}
 	} catch (error) {
 		if (error.code === LOAD_TIMEOUT_ERROR && options.browserWaitUntilFallback && options.browserWaitUntil) {
 			const browserWaitUntil = NETWORK_STATES[(NETWORK_STATES.indexOf(options.browserWaitUntil) + 1)];
@@ -212,6 +126,111 @@ async function getPageData(options) {
 	}
 }
 
+async function navigate({ Page, Runtime }, options) {
+	await Runtime.enable();
+	await Page.enable();
+	try {
+		const loadTimeoutAbortController = new AbortController();
+		const loadTimeoutAbortSignal = loadTimeoutAbortController.signal;
+		const [contextId] = await Promise.race([
+			Promise.all([getTopFrameContextId({ Page, Runtime }, options), Page.navigate({ url: options.url })]),
+			options.browserLoadMaxTime ? waitForLoadTimeout(loadTimeoutAbortSignal, options.browserLoadMaxTime) : Promise.resolve()
+		]);
+		if (!loadTimeoutAbortSignal.aborted) {
+			loadTimeoutAbortController.abort();
+		}
+		return contextId;
+	} finally {
+		await Runtime.disable();
+		await Page.disable();
+	}
+}
+
+async function getTopFrameContextId({ Page, Runtime }, options) {
+	let frameId, contextId;
+	await new Promise(resolve => {
+		const FRAME_NAVIGATED_EVENT = "frameNavigated";
+		const CONTEXT_CREATED_EVENT = "executionContextCreated";
+		Page.addEventListener(FRAME_NAVIGATED_EVENT, onFrameNavigated);
+		Runtime.addEventListener(CONTEXT_CREATED_EVENT, executionContextCreated);
+
+		async function onFrameNavigated({ params }) {
+			const { frame } = params;
+			if (!frame.parentId) {
+				frameId = frame.id;
+				await waitForPageReady({ Page }, frameId, options);
+				Page.removeEventListener(FRAME_NAVIGATED_EVENT, onFrameNavigated);
+				Runtime.removeEventListener(CONTEXT_CREATED_EVENT, executionContextCreated);
+				resolve();
+			}
+		}
+
+		function executionContextCreated({ params }) {
+			const { context } = params;
+			const { name, origin, auxData } = context;
+			if (frameId !== undefined && origin === "://" && name === SINGLE_FILE_WORLD_NAME && auxData && auxData.frameId === frameId) {
+				contextId = context.id;
+			}
+		}
+	});
+	return contextId;
+}
+
+function waitForLoadTimeout(abortSignal, maxDelay) {
+	return new Promise((resolve, reject) => {
+		const ABORT_EVENT = "abort";
+		abortSignal.addEventListener(ABORT_EVENT, onAbort);
+		const timeoutId = setTimeout(() => {
+			const error = new Error("Load timeout");
+			error.code = LOAD_TIMEOUT_ERROR;
+			reject(error);
+		}, maxDelay);
+
+		function onAbort() {
+			abortSignal.removeEventListener(ABORT_EVENT, onAbort);
+			clearTimeout(timeoutId);
+			resolve();
+		}
+	});
+}
+
+async function waitForPageReady({ Page }, topFrameId, options) {
+	await Page.setLifecycleEventsEnabled({ enabled: true });
+	await new Promise(resolve => {
+		const LIFE_CYCLE_EVENT = "lifecycleEvent";
+		Page.addEventListener(LIFE_CYCLE_EVENT, onLifecycleEvent);
+
+		function onLifecycleEvent({ params }) {
+			const { name, frameId } = params;
+			if (frameId === topFrameId) {
+				if ((name === options.browserWaitUntil || name === INTERACTIVE_TIME_STATE)) {
+					Page.removeEventListener(LIFE_CYCLE_EVENT, onLifecycleEvent);
+					resolve();
+				}
+			}
+		}
+	});
+	await Page.setLifecycleEventsEnabled({ enabled: false });
+	if (options.browserWaitDelay) {
+		setTimeout(() => resolve, options.browserWaitDelay);
+	}
+}
+
+async function waitForDebuggerReady({ Debugger }) {
+	await Debugger.enable();
+	await Debugger.pause();
+	await new Promise(resolve => {
+		const RESUMED_EVENT = "resumed";
+		Debugger.addEventListener(RESUMED_EVENT, onResumed);
+
+		function onResumed() {
+			Debugger.removeEventListener(RESUMED_EVENT, onResumed);
+			resolve();
+		}
+	});
+	await Debugger.disable();
+}
+
 function getBrowserOptions(options) {
 	const browserOptions = {};
 	browserOptions.args = options.browserArgs;

package/lib/version.js

@@ -1 +1 @@
-export const version = "2.0.17";
+export const version = "2.0.19";

package/options.js

@@ -49,7 +49,7 @@ const OPTIONS_INFO = {
 	"browser-height": { description: "Height of the browser viewport in pixels", type: "number", defaultValue: 720 },
 	"browser-load-max-time": { description: "Maximum delay of time to wait for page loading in ms", type: "number", defaultValue: 60000 },
 	"browser-wait-delay": { description: "Time to wait before capturing the page in ms", type: "number" },
-	"browser-wait-until": { description: "When to consider the page is loaded (netWorkIdle, netWorkAlmostIdle, load, domContentLoaded)", type: "string", defaultValue: "networkIdle" },
+	"browser-wait-until": { description: "When to consider the page is loaded (InteractiveTime, networkIdle, networkAlmostIdle, load, domContentLoaded)", type: "string", defaultValue: "networkIdle" },
 	"browser-wait-until-fallback": { description: "Retry with the next value of --browser-wait-until when a timeout error is thrown", type: "boolean", defaultValue: true },
 	"browser-debug": { description: "Enable debug mode", type: "boolean" },
 	"browser-script": { description: "Path of a script executed in the page (and all the frames) before it is loaded", type: "string[]" },

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "2.0.17",
+	"version": "2.0.19",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"engines": {