single-file-cli 2.0.19 → 2.0.21

package/README.MD

@@ -80,9 +80,6 @@ Make sure Chrome or a Chromium-based browser is installed in the default folder.
 
     ```sh
     unzip master.zip .
-    ```
-
-    ```sh
     cd single-file-cli-master
     ```
 
@@ -90,9 +87,6 @@ Make sure Chrome or a Chromium-based browser is installed in the default folder.
 
     ```sh
     git clone --depth 1 --recursive https://github.com/gildas-lormeau/single-file-cli.git
-    ```
-
-    ```sh
     cd single-file-cli
     ```
 

package/deno.json

@@ -1,13 +1,13 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.0.19",
+  "version": "2.0.21",
   "description": "SingleFile CLI",
   "nodeModulesDir": true,
   "exports": {
     ".": "./single-file-cli-api.js"
   },
   "imports": {
-    "simple-cdp": "jsr:@simple-cdp/simple-cdp@^1.8.2",
+    "simple-cdp": "jsr:@simple-cdp/simple-cdp@^1.8.4",
     "path": "jsr:@std/path@^0.220.1"
   }
 }

package/deno.lock

@@ -2,15 +2,15 @@
   "version": "3",
   "packages": {
     "specifiers": {
-      "jsr:@simple-cdp/simple-cdp@^1.8.2": "jsr:@simple-cdp/simple-cdp@1.8.2",
+      "jsr:@simple-cdp/simple-cdp@^1.8.4": "jsr:@simple-cdp/simple-cdp@1.8.4",
       "jsr:@std/assert@^0.220.1": "jsr:@std/assert@0.220.1",
       "jsr:@std/path@^0.220.1": "jsr:@std/path@0.220.1",
-      "npm:simple-cdp@^1.8.2": "npm:simple-cdp@1.8.2",
+      "npm:simple-cdp@^1.8.4": "npm:simple-cdp@1.8.4",
       "npm:ws@^8.16.0": "npm:ws@8.16.0"
     },
     "jsr": {
-      "@simple-cdp/simple-cdp@1.8.2": {
-        "integrity": "171f82ef24c464c83f0ffc1148096a04fcd8256c6bb6b5454640614c53689165"
+      "@simple-cdp/simple-cdp@1.8.4": {
+        "integrity": "5e31ab90a552f9f1cf84e6cf0454d21c5b9dc1d89f9c1f9a750772b47bad9a4e"
       },
       "@std/assert@0.220.1": {
         "integrity": "88710d54f3afdd7a5761e7805abba1f56cd14e4b212feffeb3e73a9f77482425"
@@ -23,8 +23,8 @@
       }
     },
     "npm": {
-      "simple-cdp@1.8.2": {
-        "integrity": "sha512-Ts4RpMMuJjdp+mLB2OJeJSUyTJZw3tUDc7ZWKytK6/4zSegwzKJEIqKv0Z2UgX0DPvma0eYg10MfaceM6CYQfA==",
+      "simple-cdp@1.8.4": {
+        "integrity": "sha512-fsvMUyz13vzX8nxSPwJfdxdROGxRDdTAFX7XitVpRsKwClowsyL3NZ6kR4vR5tZfxbz31p3uoM14eFUvhaWisA==",
         "dependencies": {}
       },
       "ws@8.16.0": {
@@ -36,12 +36,12 @@
   "remote": {},
   "workspace": {
     "dependencies": [
-      "jsr:@simple-cdp/simple-cdp@^1.8.2",
+      "jsr:@simple-cdp/simple-cdp@^1.8.4",
       "jsr:@std/path@^0.220.1"
     ],
     "packageJson": {
       "dependencies": [
-        "npm:simple-cdp@^1.8.2",
+        "npm:simple-cdp@^1.8.4",
         "npm:ws@^8.16.0"
       ]
     }

package/lib/browser.js

@@ -66,13 +66,7 @@ async function launchBrowser(options = {}, indexPath = 0) {
 			if (indexPath + 1 < BROWSER_PATHS[build.os].length) {
 				await launchBrowser(options, indexPath + 1);
 			} else {
-				let message = "Chromium executable not found. ";
-				if (options.executablePath) {
-					message += "Make sure --browser-executable-path is correct.";
-				} else {
-					message += "Set the path using the --browser-executable-path option.";
-				}
-				throw new Error(message);
+				throw error;
 			}
 		} else {
 			throw error;

package/lib/cdp-client.js

@@ -37,13 +37,17 @@ const EMPTY_PAGE_URL = "about:blank";
 export { initialize, getPageData, closeBrowser };
 
 async function initialize(options) {
-	await launchBrowser(getBrowserOptions(options));
+	if (!options.browserRemoteDebuggingUrl) {
+		await launchBrowser(getBrowserOptions(options));
+	} else {
+		CDP.remoteDebuggingUrl = options.browserRemoteDebuggingUrl;
+	}
 }
 
 async function getPageData(options) {
 	let targetInfo;
 	try {
-		const targetInfo = await CDP.createTarget(EMPTY_PAGE_URL);
+		targetInfo = await CDP.createTarget(EMPTY_PAGE_URL);
 		const { Browser, Security, Page, Emulation, Fetch, Network, Runtime, Debugger } = new CDP(targetInfo);
 		if (options.browserStartMinimized) {
 			const { windowId, bounds } = await Browser.getWindowForTarget({ targetId: targetInfo.targetId });
@@ -96,16 +100,22 @@ async function getPageData(options) {
 			worldName: SINGLE_FILE_WORLD_NAME
 		});
 		const [contextId] = await Promise.all([
-			navigate({ Page, Runtime }, options),
+			loadPage({ Page, Runtime }, options),
 			options.browserDebug ? waitForDebuggerReady({ Debugger }) : Promise.resolve()
 		]);
+		if (options.browserWaitDelay) {
+			setTimeout(() => resolve, options.browserWaitDelay);
+		}
 		const { result } = await Runtime.evaluate({
 			expression: `singlefile.getPageData(${JSON.stringify(options)})`,
 			awaitPromise: true,
 			returnByValue: true,
 			contextId
 		});
-		const { value } = result;
+		const { value, subtype, description } = result;
+		if (subtype === "error") {
+			throw new Error(description);
+		}
 		if (options.compressContent) {
 			value.content = new Uint8Array(value.content);
 		}
@@ -121,12 +131,12 @@ async function getPageData(options) {
 		throw error;
 	} finally {
 		if (targetInfo && !options.browserDebug) {
-			await CDP.closeTarget(targetInfo.targetId);
+			await CDP.closeTarget(targetInfo.id);
 		}
 	}
 }
 
-async function navigate({ Page, Runtime }, options) {
+async function loadPage({ Page, Runtime }, options) {
 	await Runtime.enable();
 	await Page.enable();
 	try {
@@ -134,7 +144,7 @@ async function navigate({ Page, Runtime }, options) {
 		const loadTimeoutAbortSignal = loadTimeoutAbortController.signal;
 		const [contextId] = await Promise.race([
 			Promise.all([getTopFrameContextId({ Page, Runtime }, options), Page.navigate({ url: options.url })]),
-			options.browserLoadMaxTime ? waitForLoadTimeout(loadTimeoutAbortSignal, options.browserLoadMaxTime) : Promise.resolve()
+			waitForLoadTimeout(loadTimeoutAbortSignal, options.browserLoadMaxTime)
 		]);
 		if (!loadTimeoutAbortSignal.aborted) {
 			loadTimeoutAbortController.abort();
@@ -146,34 +156,83 @@ async function navigate({ Page, Runtime }, options) {
 	}
 }
 
-async function getTopFrameContextId({ Page, Runtime }, options) {
-	let frameId, contextId;
-	await new Promise(resolve => {
+function getTopFrameContextId({ Page, Runtime }, options) {
+	return new Promise((resolve, reject) => {
 		const FRAME_NAVIGATED_EVENT = "frameNavigated";
 		const CONTEXT_CREATED_EVENT = "executionContextCreated";
+		const CONTEXT_DESTROYED_EVENT = "executionContextDestroyed";
+		const contextIds = [];
+		let frameId, navigationAbortController;
 		Page.addEventListener(FRAME_NAVIGATED_EVENT, onFrameNavigated);
-		Runtime.addEventListener(CONTEXT_CREATED_EVENT, executionContextCreated);
+		Runtime.addEventListener(CONTEXT_CREATED_EVENT, onExecutionContextCreated);
+		Runtime.addEventListener(CONTEXT_DESTROYED_EVENT, onExecutionContextDestroyed);
 
 		async function onFrameNavigated({ params }) {
 			const { frame } = params;
 			if (!frame.parentId) {
 				frameId = frame.id;
-				await waitForPageReady({ Page }, frameId, options);
+				if (navigationAbortController) {
+					navigationAbortController.abort();
+					contextIds.length = 0;
+				}
+				navigationAbortController = new AbortController();
+				await onTopFrameNavigated(navigationAbortController.signal);
+			}
+		}
+
+		async function onTopFrameNavigated(navigationSignal) {
+			await waitForPageReady({ Page }, frameId, options, navigationSignal);
+			if (!navigationSignal.aborted) {
 				Page.removeEventListener(FRAME_NAVIGATED_EVENT, onFrameNavigated);
-				Runtime.removeEventListener(CONTEXT_CREATED_EVENT, executionContextCreated);
-				resolve();
+				Runtime.removeEventListener(CONTEXT_CREATED_EVENT, onExecutionContextCreated);
+				Runtime.removeEventListener(CONTEXT_DESTROYED_EVENT, onExecutionContextDestroyed);
+				let contextId;
+				if (contextIds.length) {
+					let contextIdIndex = 0;
+					do {
+						if (await testExecutionContext(contextIds[contextIdIndex])) {
+							contextId = contextIds[contextIdIndex];
+						}
+						contextIdIndex++;
+					} while (contextId === undefined && contextIdIndex < contextIds.length);
+				}
+				if (contextId === undefined) {
+					reject(new Error("Execution context not found"));
+				} else {
+					resolve(contextId);
+				}
 			}
 		}
 
-		function executionContextCreated({ params }) {
+		function onExecutionContextCreated({ params }) {
 			const { context } = params;
 			const { name, origin, auxData } = context;
 			if (frameId !== undefined && origin === "://" && name === SINGLE_FILE_WORLD_NAME && auxData && auxData.frameId === frameId) {
-				contextId = context.id;
+				contextIds.push(context.id);
+			}
+		}
+
+		function onExecutionContextDestroyed({ params }) {
+			const { executionContextId } = params;
+			const index = contextIds.indexOf(executionContextId);
+			if (index > -1) {
+				contextIds.splice(index, 1);
+			}
+		}
+
+		async function testExecutionContext(contextId) {
+			try {
+				const { result } = await Runtime.evaluate({
+					expression: "singlefile !== undefined",
+					contextId
+				});
+				return result.value === true;
+			} catch (error) {
+				// ignored
 			}
+			return false;
 		}
 	});
-	return contextId;
 }
 
 function waitForLoadTimeout(abortSignal, maxDelay) {
@@ -194,41 +253,46 @@ function waitForLoadTimeout(abortSignal, maxDelay) {
 	});
 }
 
-async function waitForPageReady({ Page }, topFrameId, options) {
+async function waitForPageReady({ Page }, topFrameId, options, navigationSignal) {
 	await Page.setLifecycleEventsEnabled({ enabled: true });
-	await new Promise(resolve => {
-		const LIFE_CYCLE_EVENT = "lifecycleEvent";
-		Page.addEventListener(LIFE_CYCLE_EVENT, onLifecycleEvent);
-
-		function onLifecycleEvent({ params }) {
-			const { name, frameId } = params;
-			if (frameId === topFrameId) {
-				if ((name === options.browserWaitUntil || name === INTERACTIVE_TIME_STATE)) {
-					Page.removeEventListener(LIFE_CYCLE_EVENT, onLifecycleEvent);
-					resolve();
+	try {
+		await new Promise(resolve => {
+			const LIFE_CYCLE_EVENT = "lifecycleEvent";
+			Page.addEventListener(LIFE_CYCLE_EVENT, onLifecycleEvent);
+
+			function onLifecycleEvent({ params }) {
+				const { name, frameId } = params;
+				if (frameId === topFrameId) {
+					if (name === options.browserWaitUntil || name === INTERACTIVE_TIME_STATE || navigationSignal.aborted) {
+						Page.removeEventListener(LIFE_CYCLE_EVENT, onLifecycleEvent);
+						resolve();
+					}
 				}
 			}
+		});
+	} finally {
+		if (!navigationSignal.aborted) {
+			await Page.setLifecycleEventsEnabled({ enabled: false });
 		}
-	});
-	await Page.setLifecycleEventsEnabled({ enabled: false });
-	if (options.browserWaitDelay) {
-		setTimeout(() => resolve, options.browserWaitDelay);
 	}
 }
 
 async function waitForDebuggerReady({ Debugger }) {
 	await Debugger.enable();
-	await Debugger.pause();
-	await new Promise(resolve => {
-		const RESUMED_EVENT = "resumed";
-		Debugger.addEventListener(RESUMED_EVENT, onResumed);
+	try {
+		await Debugger.pause();
+		await new Promise(resolve => {
+			const RESUMED_EVENT = "resumed";
+			Debugger.addEventListener(RESUMED_EVENT, onResumed);
 
-		function onResumed() {
-			Debugger.removeEventListener(RESUMED_EVENT, onResumed);
-			resolve();
-		}
-	});
-	await Debugger.disable();
+			function onResumed() {
+				Debugger.removeEventListener(RESUMED_EVENT, onResumed);
+				resolve();
+			}
+		});
+	} finally {
+		await Debugger.disable();
+	}
 }
 
 function getBrowserOptions(options) {

package/lib/version.js

@@ -1 +1 @@
-export const version = "2.0.19";
+export const version = "2.0.21";

package/options.js

@@ -60,6 +60,7 @@ const OPTIONS_INFO = {
 	"browser-cookie": { description: "Ordered list of cookie parameters separated by a comma (name,value,domain,path,expires,httpOnly,secure,sameSite,url)", type: "string[]" },
 	"browser-cookies-file": { description: "Path of the cookies file formatted as a JSON file or a Netscape text file", type: "string" },
 	"browser-ignore-insecure-certs": { description: "Ignore HTTPs errors", type: "boolean" },
+	"browser-remote-debugging-url": { description: "Remote debugging URL", type: "string" },
 	"compress-content": { description: "Compress the output file into a ZIP file", type: "boolean" },
 	"compress-CSS": { description: "Compress CSS stylesheets", type: "boolean" },
 	"compress-HTML": { description: "Compress HTML content", type: "boolean", defaultValue: true },

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "2.0.19",
+	"version": "2.0.21",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"engines": {
@@ -13,7 +13,7 @@
 		"single-file": "./single-file-node.js"
 	},
 	"dependencies": {
-		"simple-cdp": "^1.8.2",
+		"simple-cdp": "^1.8.4",
 		"ws": "^8.16.0"
 	}
 }

package/single-file-cli-api.js

@@ -76,7 +76,7 @@ const DEFAULT_OPTIONS = {
 const STATE_PROCESSING = "processing";
 const STATE_PROCESSED = "processed";
 
-const { readTextFile, writeTextFile, writeFile, stdout, mkdir, stat } = Deno;
+const { readTextFile, writeTextFile, writeFile, stdout, mkdir, stat, errors } = Deno;
 let tasks = [], maxParallelWorkers, sessionFilename;
 
 export { DEFAULT_OPTIONS, VALID_URL_TEST, initialize };
@@ -84,7 +84,21 @@ export { DEFAULT_OPTIONS, VALID_URL_TEST, initialize };
 async function initialize(options) {
 	options = Object.assign({}, DEFAULT_OPTIONS, options);
 	maxParallelWorkers = options.maxParallelWorkers || 8;
-	await backend.initialize(options);
+	try {
+		await backend.initialize(options);
+	} catch (error) {
+		if (error instanceof errors.NotFound) {
+			let message = "Chromium executable not found. ";
+			if (options.browserExecutablePath) {
+				message += "Make sure --browser-executable-path is correct.";
+			} else {
+				message += "Set the path using the --browser-executable-path option.";
+			}
+			throw new Error(message);
+		} else {
+			throw error;
+		}
+	}
 	if (options.crawlSyncSession || options.crawlLoadSession) {
 		try {
 			tasks = JSON.parse(await readTextFile(options.crawlSyncSession || options.crawlLoadSession));