single-file-cli 2.0.69 → 2.0.71

package/build.sh

@@ -1,6 +1,12 @@
 #!/usr/bin/env bash
 
-deno vendor "npm:single-file-core@1.5.36"
+mv package.json package.json.tmp
+mv deno.json deno.json.tmp
+mv deno.lock deno.lock.tmp
+deno install --vendor --quiet "npm:single-file-core@1.5.41"
+mv package.json.tmp package.json
+mv deno.json.tmp deno.json
+mv deno.lock.tmp deno.lock
 
 echo "
 import { build } from 'npm:esbuild';
@@ -82,5 +88,4 @@ const version = JSON.parse(await Deno.readTextFile('./deno.json')).version;
 await Deno.writeTextFile('lib/version.js', 'export const version = ' + JSON.stringify(version) + ';');
 " |  deno run --allow-read --allow-write --allow-net --allow-run --allow-env --lock=node_modules/deno.lock.tmp -
 
-rm -rf node_modules
-rm -rf vendor
+rm -rf node_modules

package/deno.json

@@ -1,13 +1,13 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.0.69",
+  "version": "2.0.71",
   "description": "SingleFile CLI",
-  "nodeModulesDir": true,
+  "nodeModulesDir": "auto",
   "exports": {
     ".": "./single-file-cli-api.js"
   },
   "imports": {
     "simple-cdp": "jsr:@simple-cdp/simple-cdp@^1.8.5",
-    "path": "jsr:@std/path@^0.220.1"
+    "path": "jsr:@std/path@^0.224.0"
   }
 }

package/deno.lock

@@ -1,43 +1,38 @@
 {
-  "version": "3",
-  "packages": {
-    "specifiers": {
-      "jsr:@simple-cdp/simple-cdp@^1.8.5": "jsr:@simple-cdp/simple-cdp@1.8.5",
-      "jsr:@std/assert@^0.220.1": "jsr:@std/assert@0.220.1",
-      "jsr:@std/path@^0.220.1": "jsr:@std/path@0.220.1",
-      "npm:simple-cdp@^1.8.5": "npm:simple-cdp@1.8.5",
-      "npm:ws@^8.16.0": "npm:ws@8.16.0"
+  "version": "4",
+  "specifiers": {
+    "jsr:@simple-cdp/simple-cdp@^1.8.5": "1.8.5",
+    "jsr:@std/assert@0.224": "0.224.0",
+    "jsr:@std/path@0.224": "0.224.0",
+    "npm:simple-cdp@^1.8.5": "1.8.5",
+    "npm:ws@^8.16.0": "8.16.0"
+  },
+  "jsr": {
+    "@simple-cdp/simple-cdp@1.8.5": {
+      "integrity": "7386ca561cd6293bb3115eadf14b8c98d1c41c35a69c79774ea70e5101159413"
+    },
+    "@std/assert@0.224.0": {
+      "integrity": "8643233ec7aec38a940a8264a6e3eed9bfa44e7a71cc6b3c8874213ff401967f"
     },
-    "jsr": {
-      "@simple-cdp/simple-cdp@1.8.5": {
-        "integrity": "7386ca561cd6293bb3115eadf14b8c98d1c41c35a69c79774ea70e5101159413"
-      },
-      "@std/assert@0.220.1": {
-        "integrity": "88710d54f3afdd7a5761e7805abba1f56cd14e4b212feffeb3e73a9f77482425"
-      },
-      "@std/path@0.220.1": {
-        "integrity": "cc63c1b5e5192e2f718dc2f365a3514fffb26cc0380959bab0f8fb5988a52f0c",
-        "dependencies": [
-          "jsr:@std/assert@^0.220.1"
-        ]
-      }
+    "@std/path@0.224.0": {
+      "integrity": "55bca6361e5a6d158b9380e82d4981d82d338ec587de02951e2b7c3a24910ee6",
+      "dependencies": [
+        "jsr:@std/assert"
+      ]
+    }
+  },
+  "npm": {
+    "simple-cdp@1.8.5": {
+      "integrity": "sha512-bpXoeCk0mwFrr+8C8vVkhCug5h4HO49GI+MZdnyLQcFVKzTlokrimTjEoc69X2Y2n7mtjzDHTriq6B/kgUeUcA=="
     },
-    "npm": {
-      "simple-cdp@1.8.5": {
-        "integrity": "sha512-bpXoeCk0mwFrr+8C8vVkhCug5h4HO49GI+MZdnyLQcFVKzTlokrimTjEoc69X2Y2n7mtjzDHTriq6B/kgUeUcA==",
-        "dependencies": {}
-      },
-      "ws@8.16.0": {
-        "integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==",
-        "dependencies": {}
-      }
+    "ws@8.16.0": {
+      "integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ=="
     }
   },
-  "remote": {},
   "workspace": {
     "dependencies": [
       "jsr:@simple-cdp/simple-cdp@^1.8.5",
-      "jsr:@std/path@^0.220.1"
+      "jsr:@std/path@0.224"
     ],
     "packageJson": {
       "dependencies": [

package/lib/cdp-client.js

@@ -28,6 +28,7 @@ import { getScriptSource, getHookScriptSource } from "./single-file-script.js";
 import { CDP, options } from "simple-cdp";
 
 const LOAD_TIMEOUT_ERROR = "ERR_LOAD_TIMEOUT";
+const CAPTURE_TIMEOUT_ERROR = "ERR_CAPTURE_TIMEOUT";
 const NETWORK_STATES = ["InteractiveTime", "networkIdle", "networkAlmostIdle", "load", "DOMContentLoaded"];
 const MINIMIZED_WINDOW_STATE = "minimized";
 const SINGLE_FILE_WORLD_NAME = "singlefile";
@@ -50,12 +51,19 @@ async function initialize(singleFileOptions) {
 }
 
 async function getPageData(options) {
+	const debugMessages = [];
 	let targetInfo;
 	try {
+		if (options.debugMessagesFile) {
+			debugMessages.push([Date.now(), ["Loading page", EMPTY_PAGE_URL]]);
+		}
 		targetInfo = await CDP.createTarget(EMPTY_PAGE_URL);
 		const consoleMessages = [];
 		const { Browser, Security, Page, Emulation, Fetch, Network, Runtime, Debugger, Console } = new CDP(targetInfo);
 		if (options.consoleMessagesFile) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Enabling console messages"]]);
+			}
 			await Console.enable();
 			Console.addEventListener("messageAdded", ({ params }) => {
 				const { message } = params;
@@ -65,13 +73,22 @@ async function getPageData(options) {
 		if (options.browserStartMinimized) {
 			const { windowId, bounds } = await Browser.getWindowForTarget({ targetId: targetInfo.id });
 			if (bounds.windowState !== MINIMIZED_WINDOW_STATE) {
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Minimizing window"]]);
+				}
 				await Browser.setWindowBounds({ windowId, bounds: { windowState: MINIMIZED_WINDOW_STATE } });
 			}
 		}
 		if (options.browserIgnoreHTTPSErrors !== undefined && options.browserIgnoreHTTPSErrors) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Ignoring HTTPS errors"]]);
+			}
 			await Security.setIgnoreCertificateErrors({ ignore: true });
 		}
 		if (options.browserByPassCSP === undefined || options.browserByPassCSP) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Bypassing CSP"]]);
+			}
 			await Page.setBypassCSP({ enabled: true });
 		}
 		if (options.browserMobileEmulation || options.browserDeviceWidth || options.browserDeviceHeight || options.browserDeviceScaleFactor || options.platform || options.acceptLanguage) {
@@ -91,12 +108,16 @@ async function getPageData(options) {
 					const { result } = await Runtime.evaluate({ expression: "window.devicePixelRatio" });
 					browserDeviceScaleFactor = result.value;
 				}
-				await Emulation.setDeviceMetricsOverride({
+				const deviceMetricsOptions = {
 					mobile: Boolean(options.browserMobileEmulation),
 					width: options.browserDeviceWidth || (options.browserMobileEmulation ? 360 : options.width || browserDeviceWidth),
 					height: options.browserDeviceHeight || (options.browserMobileEmulation ? 800 : options.height || browserDeviceHeight),
 					deviceScaleFactor: options.browserDeviceScaleFactor || (options.browserMobileEmulation ? 2 : browserDeviceScaleFactor)
-				});
+				};
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Emulating device metrics", JSON.stringify(deviceMetricsOptions)]]);
+				}
+				await Emulation.setDeviceMetricsOverride(deviceMetricsOptions);
 			}
 			if (options.browserMobileEmulation || options.platform || options.acceptLanguage) {
 				const { userAgent, product } = await Browser.getVersion();
@@ -109,21 +130,29 @@ async function getPageData(options) {
 				if (options.platform || options.browserMobileEmulation) {
 					agentOptions.platform = options.platform || "Android";
 				}
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Emulating user agent", JSON.stringify(agentOptions)]]);
+				}
 				await Emulation.setUserAgentOverride(agentOptions);
 			}
 		}
 		const handleAuthRequests = Boolean(options.httpProxyUsername);
-		const patterns = handleAuthRequests ? [{ requestStage: "Request" }, { requestStage: "Response", resourceType: "Document" }] : [{ requestStage: "Response", resourceType: "Document" }];
+		const patterns = handleAuthRequests ? [{ requestStage: "Request" }, { requestStage: "Response" }] : [{ requestStage: "Response" }];
 		await Fetch.enable({ handleAuthRequests, patterns });
 		if (handleAuthRequests) {
-			Fetch.addEventListener("authRequired", ({ params }) => Fetch.continueWithAuth({
-				requestId: params.requestId,
-				authChallengeResponse: {
-					response: "ProvideCredentials",
-					username: options.httpProxyUsername,
-					password: options.httpProxyPassword
+			Fetch.addEventListener("authRequired", async ({ params }) => {
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Authenticating"]]);
 				}
-			}));
+				await Fetch.continueWithAuth({
+					requestId: params.requestId,
+					authChallengeResponse: {
+						response: "ProvideCredentials",
+						username: options.httpProxyUsername,
+						password: options.httpProxyPassword
+					}
+				});
+			});
 		}
 		let url = options.url;
 		let alternativeUrl = new URL(url);
@@ -134,12 +163,15 @@ async function getPageData(options) {
 		let httpInfo;
 		Fetch.addEventListener("requestPaused", async ({ params }) => {
 			const { requestId, request, resourceType, responseHeaders, responseStatusCode, responseStatusText } = params;
-			if (options.outputJson && !httpInfo && (request.url == url || request.url == alternativeUrl) && responseStatusCode !== undefined) {
+			if (resourceType == "Document" && (options.outputJson && !httpInfo && (request.url == url || request.url == alternativeUrl) && responseStatusCode !== undefined)) {
 				if (REDIRECT_STATUS_CODES.includes(responseStatusCode)) {
 					const redirect = responseHeaders.find(header => header.name.toLowerCase() == "location").value;
 					if (redirect) {
 						url = new URL(redirect, url).href;
 					}
+					if (options.debugMessagesFile) {
+						debugMessages.push([Date.now(), ["Redirecting", url]]);
+					}
 				} else {
 					httpInfo = {
 						request: {
@@ -156,6 +188,18 @@ async function getPageData(options) {
 						}
 					};
 				}
+			} else if (options.blockedURLPatterns && options.blockedURLPatterns.length) {
+				const blockedURL = options.blockedURLPatterns.find(blockedURL => new RegExp(blockedURL).test(request.url));
+				if (blockedURL) {
+					try {
+						if (options.debugMessagesFile) {
+							debugMessages.push([Date.now(), ["Blocking request", request.url]]);
+						}
+						await Fetch.failRequest({ requestId, errorReason: "Aborted" });
+					} catch {
+						// ignored
+					}
+				}
 			}
 			try {
 				await Fetch.continueRequest({ requestId });
@@ -164,10 +208,16 @@ async function getPageData(options) {
 			}
 		});
 		if (options.httpHeaders && options.httpHeaders.length) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Setting HTTP headers", JSON.stringify(options.httpHeaders)]]);
+			}
 			await Network.setExtraHTTPHeaders({ headers: options.httpHeaders });
 		}
 		if (options.emulateMediaFeatures) {
 			for (const mediaFeature of options.emulateMediaFeatures) {
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Emulating media feature", mediaFeature.name, mediaFeature.value]]);
+				}
 				await Emulation.setEmulatedMedia({
 					media: mediaFeature.name,
 					features: mediaFeature.value.split(",").map(feature => feature.trim())
@@ -175,6 +225,9 @@ async function getPageData(options) {
 			}
 		}
 		if (options.browserCookies && options.browserCookies.length) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Setting cookies", JSON.stringify(options.browserCookies)]]);
+			}
 			await Network.setCookies({ cookies: options.browserCookies });
 		}
 		await Browser.setDownloadBehavior({ behavior: "deny" });
@@ -188,7 +241,7 @@ async function getPageData(options) {
 			worldName: SINGLE_FILE_WORLD_NAME
 		});
 		const [contextId] = await Promise.all([
-			loadPage({ Page, Runtime }, options),
+			loadPage({ Page, Runtime }, options, debugMessages),
 			options.browserDebug ? waitForDebuggerReady({ Debugger }) : Promise.resolve()
 		]);
 		await Runtime.addBinding({ name: SET_PAGE_DATA_FUNCTION_NAME, executionContextId: contextId });
@@ -196,6 +249,9 @@ async function getPageData(options) {
 			await Runtime.addBinding({ name: CAPTURE_SCREENSHOT_FUNCTION_NAME, executionContextId: contextId });
 			Runtime.addEventListener("bindingCalled", async ({ params }) => {
 				if (params.name === CAPTURE_SCREENSHOT_FUNCTION_NAME) {
+					if (options.debugMessagesFile) {
+						debugMessages.push([Date.now(), ["Capturing screenshot"]]);
+					}
 					try {
 						let screenshotOptions = { captureBeyondViewport: true };
 						if (options.embedScreenshotOptions) {
@@ -218,6 +274,9 @@ async function getPageData(options) {
 			await Runtime.addBinding({ name: PRINT_TO_PDF_FUNCTION_NAME, executionContextId: contextId });
 			Runtime.addEventListener("bindingCalled", async ({ params }) => {
 				if (params.name === PRINT_TO_PDF_FUNCTION_NAME) {
+					if (options.debugMessagesFile) {
+						debugMessages.push([Date.now(), ["Printing to PDF", options.embedPdfOptions || ""]]);
+					}
 					let pdfOptions = {};
 					if (options.embedPdfOptions) {
 						try {
@@ -243,6 +302,9 @@ async function getPageData(options) {
 					if (payload.length) {
 						pageDataResponse += payload;
 					} else {
+						if (options.debugMessagesFile) {
+							debugMessages.push([Date.now(), ["Setting page data"]]);
+						}
 						const result = JSON.parse(pageDataResponse);
 						if (result.content instanceof Array) {
 							result.content = new Uint8Array(result.content);
@@ -253,36 +315,71 @@ async function getPageData(options) {
 			});
 		});
 		if (options.browserWaitDelay) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), [`Waiting ${options.browserWaitDelay} ms`]]);
+			}
 			await new Promise(resolve => setTimeout(resolve, options.browserWaitDelay));
 		}
-		const { result } = await Runtime.evaluate({
-			expression: `(${getPageDataScriptSource.toString()})(${JSON.stringify(options)},${JSON.stringify([SET_SCREENSHOT_FUNCTION_NAME, SET_PDF_FUNCTION_NAME, SET_PAGE_DATA_FUNCTION_NAME, CAPTURE_SCREENSHOT_FUNCTION_NAME, PRINT_TO_PDF_FUNCTION_NAME])})`,
-			awaitPromise: true,
-			returnByValue: true,
-			contextId
-		});
-		const { subtype, description } = result;
-		if (subtype === "error") {
-			throw new Error(description);
+		const captureTimeoutAbortController = new AbortController();
+		const captureTimeoutAbortSignal = captureTimeoutAbortController.signal;
+		try {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Capturing page"]]);
+			}
+			const { result } = await Promise.race([
+				Runtime.evaluate({
+					expression: `(${getPageDataScriptSource.toString()})(${JSON.stringify(options)},${JSON.stringify([SET_SCREENSHOT_FUNCTION_NAME, SET_PDF_FUNCTION_NAME, SET_PAGE_DATA_FUNCTION_NAME, CAPTURE_SCREENSHOT_FUNCTION_NAME, PRINT_TO_PDF_FUNCTION_NAME])})`,
+					awaitPromise: true,
+					returnByValue: true,
+					contextId
+				}),
+				waitForTimeout(captureTimeoutAbortSignal, options.browserCaptureMaxTime, "Capture timeout", CAPTURE_TIMEOUT_ERROR)
+			]);
+			const { subtype, description } = result;
+			if (subtype === "error") {
+				throw new Error(description);
+			}
+		} finally {
+			if (!captureTimeoutAbortSignal.aborted) {
+				captureTimeoutAbortController.abort();
+			}
+			await Runtime.disable();
+			await Page.disable();
 		}
 		const pageData = await pageDataPromise;
 		if (options.consoleMessagesFile) {
 			await Console.disable();
 			pageData.consoleMessages = consoleMessages;
 		}
+		if (options.debugMessagesFile) {
+			pageData.debugMessages = debugMessages;
+			debugMessages.push([Date.now(), ["Returning page data"]]);
+		}
 		Object.assign(pageData, httpInfo);
 		return pageData;
 	} catch (error) {
 		if (error.code === LOAD_TIMEOUT_ERROR && options.browserWaitUntilFallback && options.browserWaitUntil) {
 			const browserWaitUntil = NETWORK_STATES[(NETWORK_STATES.indexOf(options.browserWaitUntil) + 1)];
 			if (browserWaitUntil) {
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Retrying with waitUntil", browserWaitUntil]]);
+				}
 				options.browserWaitUntil = browserWaitUntil;
 				await closeTarget();
 				return await getPageData(options);
 			}
 		}
+		if (options.consoleMessagesFile) {
+			error.consoleMessages = consoleMessages;
+		}
+		if (options.debugMessagesFile) {
+			error.debugMessages = debugMessages;
+		}
 		throw error;
 	} finally {
+		if (options.debugMessagesFile) {
+			debugMessages.push([Date.now(), ["Closing page"]]);
+		}
 		await closeTarget();
 	}
 
@@ -335,15 +432,18 @@ function getPageDataScriptSource(options, [SET_SCREENSHOT_FUNCTION_NAME, SET_PDF
 	});
 }
 
-async function loadPage({ Page, Runtime }, options) {
+async function loadPage({ Page, Runtime }, options, debugMessages) {
 	await Runtime.enable();
 	await Page.enable();
 	const loadTimeoutAbortController = new AbortController();
 	const loadTimeoutAbortSignal = loadTimeoutAbortController.signal;
 	try {
+		if (options.debugMessagesFile) {
+			debugMessages.push([Date.now(), ["Loading page", options.url]]);
+		}
 		const [contextId] = await Promise.race([
-			Promise.all([getTopFrameContextId({ Page, Runtime }, options), Page.navigate({ url: options.url })]),
-			waitForLoadTimeout(loadTimeoutAbortSignal, options.browserLoadMaxTime)
+			Promise.all([getTopFrameContextId({ Page, Runtime }, options, debugMessages), Page.navigate({ url: options.url })]),
+			waitForTimeout(loadTimeoutAbortSignal, options.browserLoadMaxTime, "Load timeout", LOAD_TIMEOUT_ERROR)
 		]);
 		return contextId;
 	} finally {
@@ -355,13 +455,16 @@ async function loadPage({ Page, Runtime }, options) {
 	}
 }
 
-async function getTopFrameContextId({ Page, Runtime }, options) {
+async function getTopFrameContextId({ Page, Runtime }, options, debugMessages) {
 	const CONTEXT_CREATED_EVENT = "executionContextCreated";
 	const contextIds = [];
 	let topFrameId;
 	try {
 		Runtime.addEventListener(CONTEXT_CREATED_EVENT, onExecutionContextCreated);
 		await waitForPageReady({ Page }, options);
+		if (options.debugMessagesFile) {
+			debugMessages.push([Date.now(), ["Getting execution context"]]);
+		}
 		const contextId = await getContextId();
 		if (contextId === undefined) {
 			throw new Error("Execution context not found");
@@ -392,12 +495,23 @@ async function getTopFrameContextId({ Page, Runtime }, options) {
 
 				function onLifecycleEvent({ params }) {
 					const { frameId, name } = params;
-					if (frameId === topFrameId && name === options.browserWaitUntil || (resolveCallbackTimeout && NETWORK_STATES.indexOf(name) < NETWORK_STATES.indexOf(options.browserWaitUntil))) {
-						clearTimeout(resolveCallbackTimeout);
-						setTimeout(() => {
-							removeListeners();
-							resolve();
-						}, options.browserWaitUntilDelay);
+					if (frameId === topFrameId) {
+						if (options.debugMessagesFile) {
+							debugMessages.push([Date.now(), ["Detecting lifecycle event", name]]);
+						}
+						if (name === options.browserWaitUntil || (resolveCallbackTimeout && NETWORK_STATES.indexOf(name) < NETWORK_STATES.indexOf(options.browserWaitUntil))) {
+							clearTimeout(resolveCallbackTimeout);
+							if (options.debugMessagesFile) {
+								debugMessages.push([Date.now(), [`Waiting ${options.browserWaitUntilDelay} ms`]]);
+							}
+							setTimeout(() => {
+								if (options.debugMessagesFile) {
+									debugMessages.push([Date.now(), ["Detecting page ready"]]);
+								}
+								removeListeners();
+								resolve();
+							}, options.browserWaitUntilDelay);
+						}
 					}
 				}
 
@@ -409,6 +523,9 @@ async function getTopFrameContextId({ Page, Runtime }, options) {
 							removeListeners();
 							reject(new Error("Unreachable URL: " + frame.unreachableUrl));
 						} else {
+							if (options.debugMessagesFile) {
+								debugMessages.push([Date.now(), ["Detecting top frame ID"]]);
+							}
 							topFrameId = frame.id;
 						}
 					}
@@ -452,14 +569,14 @@ async function getTopFrameContextId({ Page, Runtime }, options) {
 	}
 }
 
-function waitForLoadTimeout(abortSignal, maxDelay) {
+function waitForTimeout(abortSignal, maxDelay, errorMessage, errorCode) {
 	return new Promise((resolve, reject) => {
 		const ABORT_EVENT = "abort";
 		abortSignal.addEventListener(ABORT_EVENT, onAbort);
 		const timeoutId = setTimeout(() => {
 			abortSignal.removeEventListener(ABORT_EVENT, onAbort);
-			const error = new Error("Load timeout");
-			error.code = LOAD_TIMEOUT_ERROR;
+			const error = new Error(errorMessage);
+			error.code = errorCode;
 			reject(error);
 		}, maxDelay);
 

package/lib/constants.js

@@ -51,7 +51,9 @@ const BROWSER_ARGS = [
 	"--enable-use-zoom-for-dsf=false",
 	"--no-sandbox",
 	"--no-startup-window",
-	"--bwsi"
+	"--bwsi",
+	"--mute-audio",
+	"--deny-permission-prompts"
 ];
 const BROWSER_PATHS = {
 	linux: [