single-file-cli 2.0.69 → 2.0.70

package/build.sh

@@ -1,6 +1,12 @@
 #!/usr/bin/env bash
 
-deno vendor "npm:single-file-core@1.5.36"
+mv package.json package.json.tmp
+mv deno.json deno.json.tmp
+mv deno.lock deno.lock.tmp
+deno install --vendor --quiet "npm:single-file-core@1.5.40"
+mv package.json.tmp package.json
+mv deno.json.tmp deno.json
+mv deno.lock.tmp deno.lock
 
 echo "
 import { build } from 'npm:esbuild';
@@ -82,5 +88,4 @@ const version = JSON.parse(await Deno.readTextFile('./deno.json')).version;
 await Deno.writeTextFile('lib/version.js', 'export const version = ' + JSON.stringify(version) + ';');
 " |  deno run --allow-read --allow-write --allow-net --allow-run --allow-env --lock=node_modules/deno.lock.tmp -
 
-rm -rf node_modules
-rm -rf vendor
+rm -rf node_modules

package/deno.json

@@ -1,8 +1,8 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.0.69",
+  "version": "2.0.70",
   "description": "SingleFile CLI",
-  "nodeModulesDir": true,
+  "nodeModulesDir": "auto",
   "exports": {
     ".": "./single-file-cli-api.js"
   },

package/lib/cdp-client.js

@@ -50,12 +50,19 @@ async function initialize(singleFileOptions) {
 }
 
 async function getPageData(options) {
+	const debugMessages = [];
 	let targetInfo;
 	try {
+		if (options.debugMessagesFile) {
+			debugMessages.push([Date.now(), ["Loading page", EMPTY_PAGE_URL]]);
+		}
 		targetInfo = await CDP.createTarget(EMPTY_PAGE_URL);
 		const consoleMessages = [];
 		const { Browser, Security, Page, Emulation, Fetch, Network, Runtime, Debugger, Console } = new CDP(targetInfo);
 		if (options.consoleMessagesFile) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Enabling console messages"]]);
+			}
 			await Console.enable();
 			Console.addEventListener("messageAdded", ({ params }) => {
 				const { message } = params;
@@ -65,13 +72,22 @@ async function getPageData(options) {
 		if (options.browserStartMinimized) {
 			const { windowId, bounds } = await Browser.getWindowForTarget({ targetId: targetInfo.id });
 			if (bounds.windowState !== MINIMIZED_WINDOW_STATE) {
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Minimizing window"]]);
+				}
 				await Browser.setWindowBounds({ windowId, bounds: { windowState: MINIMIZED_WINDOW_STATE } });
 			}
 		}
 		if (options.browserIgnoreHTTPSErrors !== undefined && options.browserIgnoreHTTPSErrors) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Ignoring HTTPS errors"]]);
+			}
 			await Security.setIgnoreCertificateErrors({ ignore: true });
 		}
 		if (options.browserByPassCSP === undefined || options.browserByPassCSP) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Bypassing CSP"]]);
+			}
 			await Page.setBypassCSP({ enabled: true });
 		}
 		if (options.browserMobileEmulation || options.browserDeviceWidth || options.browserDeviceHeight || options.browserDeviceScaleFactor || options.platform || options.acceptLanguage) {
@@ -91,12 +107,16 @@ async function getPageData(options) {
 					const { result } = await Runtime.evaluate({ expression: "window.devicePixelRatio" });
 					browserDeviceScaleFactor = result.value;
 				}
-				await Emulation.setDeviceMetricsOverride({
+				const deviceMetricsOptions = {
 					mobile: Boolean(options.browserMobileEmulation),
 					width: options.browserDeviceWidth || (options.browserMobileEmulation ? 360 : options.width || browserDeviceWidth),
 					height: options.browserDeviceHeight || (options.browserMobileEmulation ? 800 : options.height || browserDeviceHeight),
 					deviceScaleFactor: options.browserDeviceScaleFactor || (options.browserMobileEmulation ? 2 : browserDeviceScaleFactor)
-				});
+				};
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Emulating device metrics", JSON.stringify(deviceMetricsOptions)]]);
+				}
+				await Emulation.setDeviceMetricsOverride(deviceMetricsOptions);
 			}
 			if (options.browserMobileEmulation || options.platform || options.acceptLanguage) {
 				const { userAgent, product } = await Browser.getVersion();
@@ -109,21 +129,29 @@ async function getPageData(options) {
 				if (options.platform || options.browserMobileEmulation) {
 					agentOptions.platform = options.platform || "Android";
 				}
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Emulating user agent", JSON.stringify(agentOptions)]]);
+				}
 				await Emulation.setUserAgentOverride(agentOptions);
 			}
 		}
 		const handleAuthRequests = Boolean(options.httpProxyUsername);
-		const patterns = handleAuthRequests ? [{ requestStage: "Request" }, { requestStage: "Response", resourceType: "Document" }] : [{ requestStage: "Response", resourceType: "Document" }];
+		const patterns = handleAuthRequests ? [{ requestStage: "Request" }, { requestStage: "Response" }] : [{ requestStage: "Response" }];
 		await Fetch.enable({ handleAuthRequests, patterns });
 		if (handleAuthRequests) {
-			Fetch.addEventListener("authRequired", ({ params }) => Fetch.continueWithAuth({
-				requestId: params.requestId,
-				authChallengeResponse: {
-					response: "ProvideCredentials",
-					username: options.httpProxyUsername,
-					password: options.httpProxyPassword
+			Fetch.addEventListener("authRequired", async ({ params }) => {
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Authenticating"]]);
 				}
-			}));
+				await Fetch.continueWithAuth({
+					requestId: params.requestId,
+					authChallengeResponse: {
+						response: "ProvideCredentials",
+						username: options.httpProxyUsername,
+						password: options.httpProxyPassword
+					}
+				});
+			});
 		}
 		let url = options.url;
 		let alternativeUrl = new URL(url);
@@ -134,12 +162,15 @@ async function getPageData(options) {
 		let httpInfo;
 		Fetch.addEventListener("requestPaused", async ({ params }) => {
 			const { requestId, request, resourceType, responseHeaders, responseStatusCode, responseStatusText } = params;
-			if (options.outputJson && !httpInfo && (request.url == url || request.url == alternativeUrl) && responseStatusCode !== undefined) {
+			if (resourceType == "Document" && (options.outputJson && !httpInfo && (request.url == url || request.url == alternativeUrl) && responseStatusCode !== undefined)) {
 				if (REDIRECT_STATUS_CODES.includes(responseStatusCode)) {
 					const redirect = responseHeaders.find(header => header.name.toLowerCase() == "location").value;
 					if (redirect) {
 						url = new URL(redirect, url).href;
 					}
+					if (options.debugMessagesFile) {
+						debugMessages.push([Date.now(), ["Redirecting", url]]);
+					}
 				} else {
 					httpInfo = {
 						request: {
@@ -156,6 +187,18 @@ async function getPageData(options) {
 						}
 					};
 				}
+			} else if (options.blockedURLPatterns && options.blockedURLPatterns.length) {
+				const blockedURL = options.blockedURLPatterns.find(blockedURL => new RegExp(blockedURL).test(request.url));
+				if (blockedURL) {
+					try {
+						if (options.debugMessagesFile) {
+							debugMessages.push([Date.now(), ["Blocking request", request.url]]);
+						}
+						await Fetch.failRequest({ requestId, errorReason: "Aborted" });
+					} catch {
+						// ignored
+					}
+				}
 			}
 			try {
 				await Fetch.continueRequest({ requestId });
@@ -164,10 +207,16 @@ async function getPageData(options) {
 			}
 		});
 		if (options.httpHeaders && options.httpHeaders.length) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Setting HTTP headers", JSON.stringify(options.httpHeaders)]]);
+			}
 			await Network.setExtraHTTPHeaders({ headers: options.httpHeaders });
 		}
 		if (options.emulateMediaFeatures) {
 			for (const mediaFeature of options.emulateMediaFeatures) {
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Emulating media feature", mediaFeature.name, mediaFeature.value]]);
+				}
 				await Emulation.setEmulatedMedia({
 					media: mediaFeature.name,
 					features: mediaFeature.value.split(",").map(feature => feature.trim())
@@ -175,6 +224,9 @@ async function getPageData(options) {
 			}
 		}
 		if (options.browserCookies && options.browserCookies.length) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Setting cookies", JSON.stringify(options.browserCookies)]]);
+			}
 			await Network.setCookies({ cookies: options.browserCookies });
 		}
 		await Browser.setDownloadBehavior({ behavior: "deny" });
@@ -188,7 +240,7 @@ async function getPageData(options) {
 			worldName: SINGLE_FILE_WORLD_NAME
 		});
 		const [contextId] = await Promise.all([
-			loadPage({ Page, Runtime }, options),
+			loadPage({ Page, Runtime }, options, debugMessages),
 			options.browserDebug ? waitForDebuggerReady({ Debugger }) : Promise.resolve()
 		]);
 		await Runtime.addBinding({ name: SET_PAGE_DATA_FUNCTION_NAME, executionContextId: contextId });
@@ -196,6 +248,9 @@ async function getPageData(options) {
 			await Runtime.addBinding({ name: CAPTURE_SCREENSHOT_FUNCTION_NAME, executionContextId: contextId });
 			Runtime.addEventListener("bindingCalled", async ({ params }) => {
 				if (params.name === CAPTURE_SCREENSHOT_FUNCTION_NAME) {
+					if (options.debugMessagesFile) {
+						debugMessages.push([Date.now(), ["Capturing screenshot"]]);
+					}
 					try {
 						let screenshotOptions = { captureBeyondViewport: true };
 						if (options.embedScreenshotOptions) {
@@ -218,6 +273,9 @@ async function getPageData(options) {
 			await Runtime.addBinding({ name: PRINT_TO_PDF_FUNCTION_NAME, executionContextId: contextId });
 			Runtime.addEventListener("bindingCalled", async ({ params }) => {
 				if (params.name === PRINT_TO_PDF_FUNCTION_NAME) {
+					if (options.debugMessagesFile) {
+						debugMessages.push([Date.now(), ["Printing to PDF", options.embedPdfOptions || ""]]);
+					}
 					let pdfOptions = {};
 					if (options.embedPdfOptions) {
 						try {
@@ -243,6 +301,9 @@ async function getPageData(options) {
 					if (payload.length) {
 						pageDataResponse += payload;
 					} else {
+						if (options.debugMessagesFile) {
+							debugMessages.push([Date.now(), ["Setting page data"]]);
+						}
 						const result = JSON.parse(pageDataResponse);
 						if (result.content instanceof Array) {
 							result.content = new Uint8Array(result.content);
@@ -253,8 +314,14 @@ async function getPageData(options) {
 			});
 		});
 		if (options.browserWaitDelay) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), [`Waiting ${options.browserWaitDelay} ms`]]);
+			}
 			await new Promise(resolve => setTimeout(resolve, options.browserWaitDelay));
 		}
+		if (options.debugMessagesFile) {
+			debugMessages.push([Date.now(), ["Capturing page"]]);
+		}
 		const { result } = await Runtime.evaluate({
 			expression: `(${getPageDataScriptSource.toString()})(${JSON.stringify(options)},${JSON.stringify([SET_SCREENSHOT_FUNCTION_NAME, SET_PDF_FUNCTION_NAME, SET_PAGE_DATA_FUNCTION_NAME, CAPTURE_SCREENSHOT_FUNCTION_NAME, PRINT_TO_PDF_FUNCTION_NAME])})`,
 			awaitPromise: true,
@@ -270,19 +337,35 @@ async function getPageData(options) {
 			await Console.disable();
 			pageData.consoleMessages = consoleMessages;
 		}
+		if (options.debugMessagesFile) {
+			pageData.debugMessages = debugMessages;
+			debugMessages.push([Date.now(), ["Returning page data"]]);
+		}
 		Object.assign(pageData, httpInfo);
 		return pageData;
 	} catch (error) {
 		if (error.code === LOAD_TIMEOUT_ERROR && options.browserWaitUntilFallback && options.browserWaitUntil) {
 			const browserWaitUntil = NETWORK_STATES[(NETWORK_STATES.indexOf(options.browserWaitUntil) + 1)];
 			if (browserWaitUntil) {
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Retrying with waitUntil", browserWaitUntil]]);
+				}
 				options.browserWaitUntil = browserWaitUntil;
 				await closeTarget();
 				return await getPageData(options);
 			}
 		}
+		if (options.consoleMessagesFile) {
+			error.consoleMessages = consoleMessages;
+		}
+		if (options.debugMessagesFile) {
+			error.debugMessages = debugMessages;
+		}
 		throw error;
 	} finally {
+		if (options.debugMessagesFile) {
+			debugMessages.push([Date.now(), ["Closing page"]]);
+		}
 		await closeTarget();
 	}
 
@@ -335,14 +418,17 @@ function getPageDataScriptSource(options, [SET_SCREENSHOT_FUNCTION_NAME, SET_PDF
 	});
 }
 
-async function loadPage({ Page, Runtime }, options) {
+async function loadPage({ Page, Runtime }, options, debugMessages) {
 	await Runtime.enable();
 	await Page.enable();
 	const loadTimeoutAbortController = new AbortController();
 	const loadTimeoutAbortSignal = loadTimeoutAbortController.signal;
 	try {
+		if (options.debugMessagesFile) {
+			debugMessages.push([Date.now(), ["Loading page", options.url]]);
+		}
 		const [contextId] = await Promise.race([
-			Promise.all([getTopFrameContextId({ Page, Runtime }, options), Page.navigate({ url: options.url })]),
+			Promise.all([getTopFrameContextId({ Page, Runtime }, options, debugMessages), Page.navigate({ url: options.url })]),
 			waitForLoadTimeout(loadTimeoutAbortSignal, options.browserLoadMaxTime)
 		]);
 		return contextId;
@@ -355,13 +441,16 @@ async function loadPage({ Page, Runtime }, options) {
 	}
 }
 
-async function getTopFrameContextId({ Page, Runtime }, options) {
+async function getTopFrameContextId({ Page, Runtime }, options, debugMessages) {
 	const CONTEXT_CREATED_EVENT = "executionContextCreated";
 	const contextIds = [];
 	let topFrameId;
 	try {
 		Runtime.addEventListener(CONTEXT_CREATED_EVENT, onExecutionContextCreated);
 		await waitForPageReady({ Page }, options);
+		if (options.debugMessagesFile) {
+			debugMessages.push([Date.now(), ["Getting execution context"]]);
+		}
 		const contextId = await getContextId();
 		if (contextId === undefined) {
 			throw new Error("Execution context not found");
@@ -392,12 +481,23 @@ async function getTopFrameContextId({ Page, Runtime }, options) {
 
 				function onLifecycleEvent({ params }) {
 					const { frameId, name } = params;
-					if (frameId === topFrameId && name === options.browserWaitUntil || (resolveCallbackTimeout && NETWORK_STATES.indexOf(name) < NETWORK_STATES.indexOf(options.browserWaitUntil))) {
-						clearTimeout(resolveCallbackTimeout);
-						setTimeout(() => {
-							removeListeners();
-							resolve();
-						}, options.browserWaitUntilDelay);
+					if (frameId === topFrameId) {
+						if (options.debugMessagesFile) {
+							debugMessages.push([Date.now(), ["Detecting lifecycle event", name]]);
+						}
+						if (name === options.browserWaitUntil || (resolveCallbackTimeout && NETWORK_STATES.indexOf(name) < NETWORK_STATES.indexOf(options.browserWaitUntil))) {
+							clearTimeout(resolveCallbackTimeout);
+							if (options.debugMessagesFile) {
+								debugMessages.push([Date.now(), [`Waiting ${options.browserWaitUntilDelay} ms`]]);
+							}
+							setTimeout(() => {
+								if (options.debugMessagesFile) {
+									debugMessages.push([Date.now(), ["Detecting page ready"]]);
+								}
+								removeListeners();
+								resolve();
+							}, options.browserWaitUntilDelay);
+						}
 					}
 				}
 
@@ -409,6 +509,9 @@ async function getTopFrameContextId({ Page, Runtime }, options) {
 							removeListeners();
 							reject(new Error("Unreachable URL: " + frame.unreachableUrl));
 						} else {
+							if (options.debugMessagesFile) {
+								debugMessages.push([Date.now(), ["Detecting top frame ID"]]);
+							}
 							topFrameId = frame.id;
 						}
 					}

package/lib/constants.js

@@ -51,7 +51,9 @@ const BROWSER_ARGS = [
 	"--enable-use-zoom-for-dsf=false",
 	"--no-sandbox",
 	"--no-startup-window",
-	"--bwsi"
+	"--bwsi",
+	"--mute-audio",
+	"--deny-permission-prompts"
 ];
 const BROWSER_PATHS = {
 	linux: [