single-file-cli 2.0.68 → 2.0.70

package/README.MD

@@ -65,7 +65,7 @@ Make sure Chrome or a Chromium-based browser is installed in the default folder.
 
 - There are 3 ways to download the code of SingleFile, choose the one you prefer:
   
-  - Install with `npm` (`npm` is installed with Node.js) and run with `npx`
+  - Install with `npm` and run `single-file` via `npx` (`npm` and `npx` are installed with Node.js)
   
     ```sh
     npm install "single-file-cli"

package/build.sh

@@ -1,6 +1,12 @@
 #!/usr/bin/env bash
 
-deno vendor "npm:single-file-core@1.5.36"
+mv package.json package.json.tmp
+mv deno.json deno.json.tmp
+mv deno.lock deno.lock.tmp
+deno install --vendor --quiet "npm:single-file-core@1.5.40"
+mv package.json.tmp package.json
+mv deno.json.tmp deno.json
+mv deno.lock.tmp deno.lock
 
 echo "
 import { build } from 'npm:esbuild';
@@ -82,5 +88,4 @@ const version = JSON.parse(await Deno.readTextFile('./deno.json')).version;
 await Deno.writeTextFile('lib/version.js', 'export const version = ' + JSON.stringify(version) + ';');
 " |  deno run --allow-read --allow-write --allow-net --allow-run --allow-env --lock=node_modules/deno.lock.tmp -
 
-rm -rf node_modules
-rm -rf vendor
+rm -rf node_modules

package/deno.json

@@ -1,8 +1,8 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.0.68",
+  "version": "2.0.70",
   "description": "SingleFile CLI",
-  "nodeModulesDir": true,
+  "nodeModulesDir": "auto",
   "exports": {
     ".": "./single-file-cli-api.js"
   },

package/lib/browser.js

@@ -92,7 +92,7 @@ async function launchBrowser(options = {}, indexPath = 0) {
 async function getDebugPort(port = getRandomDebugPort(), usedPorts = []) {
 	try {
 		await fetch("http://localhost:" + port + "/json/version");
-	} catch (error) {
+	} catch {
 		return port;
 	}
 	if (usedPorts.length < DEBUG_PORT_RANGE) {
@@ -121,7 +121,7 @@ async function closeBrowser() {
 		try {
 			await remove(profilePath, { recursive: true });
 			profilePath = undefined;
-		} catch (error) {
+		} catch {
 			console.log("Warning: failed to remove profile directory: " + profilePath); // eslint-disable-line no-console
 		}
 	}

package/lib/cdp-client.js

@@ -50,12 +50,19 @@ async function initialize(singleFileOptions) {
 }
 
 async function getPageData(options) {
+	const debugMessages = [];
 	let targetInfo;
 	try {
+		if (options.debugMessagesFile) {
+			debugMessages.push([Date.now(), ["Loading page", EMPTY_PAGE_URL]]);
+		}
 		targetInfo = await CDP.createTarget(EMPTY_PAGE_URL);
 		const consoleMessages = [];
 		const { Browser, Security, Page, Emulation, Fetch, Network, Runtime, Debugger, Console } = new CDP(targetInfo);
 		if (options.consoleMessagesFile) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Enabling console messages"]]);
+			}
 			await Console.enable();
 			Console.addEventListener("messageAdded", ({ params }) => {
 				const { message } = params;
@@ -65,13 +72,22 @@ async function getPageData(options) {
 		if (options.browserStartMinimized) {
 			const { windowId, bounds } = await Browser.getWindowForTarget({ targetId: targetInfo.id });
 			if (bounds.windowState !== MINIMIZED_WINDOW_STATE) {
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Minimizing window"]]);
+				}
 				await Browser.setWindowBounds({ windowId, bounds: { windowState: MINIMIZED_WINDOW_STATE } });
 			}
 		}
 		if (options.browserIgnoreHTTPSErrors !== undefined && options.browserIgnoreHTTPSErrors) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Ignoring HTTPS errors"]]);
+			}
 			await Security.setIgnoreCertificateErrors({ ignore: true });
 		}
 		if (options.browserByPassCSP === undefined || options.browserByPassCSP) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Bypassing CSP"]]);
+			}
 			await Page.setBypassCSP({ enabled: true });
 		}
 		if (options.browserMobileEmulation || options.browserDeviceWidth || options.browserDeviceHeight || options.browserDeviceScaleFactor || options.platform || options.acceptLanguage) {
@@ -91,12 +107,16 @@ async function getPageData(options) {
 					const { result } = await Runtime.evaluate({ expression: "window.devicePixelRatio" });
 					browserDeviceScaleFactor = result.value;
 				}
-				await Emulation.setDeviceMetricsOverride({
+				const deviceMetricsOptions = {
 					mobile: Boolean(options.browserMobileEmulation),
 					width: options.browserDeviceWidth || (options.browserMobileEmulation ? 360 : options.width || browserDeviceWidth),
 					height: options.browserDeviceHeight || (options.browserMobileEmulation ? 800 : options.height || browserDeviceHeight),
 					deviceScaleFactor: options.browserDeviceScaleFactor || (options.browserMobileEmulation ? 2 : browserDeviceScaleFactor)
-				});
+				};
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Emulating device metrics", JSON.stringify(deviceMetricsOptions)]]);
+				}
+				await Emulation.setDeviceMetricsOverride(deviceMetricsOptions);
 			}
 			if (options.browserMobileEmulation || options.platform || options.acceptLanguage) {
 				const { userAgent, product } = await Browser.getVersion();
@@ -109,21 +129,29 @@ async function getPageData(options) {
 				if (options.platform || options.browserMobileEmulation) {
 					agentOptions.platform = options.platform || "Android";
 				}
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Emulating user agent", JSON.stringify(agentOptions)]]);
+				}
 				await Emulation.setUserAgentOverride(agentOptions);
 			}
 		}
 		const handleAuthRequests = Boolean(options.httpProxyUsername);
-		const patterns = handleAuthRequests ? [{ requestStage: "Request" }, { requestStage: "Response", resourceType: "Document" }] : [{ requestStage: "Response", resourceType: "Document" }];
+		const patterns = handleAuthRequests ? [{ requestStage: "Request" }, { requestStage: "Response" }] : [{ requestStage: "Response" }];
 		await Fetch.enable({ handleAuthRequests, patterns });
 		if (handleAuthRequests) {
-			Fetch.addEventListener("authRequired", ({ params }) => Fetch.continueWithAuth({
-				requestId: params.requestId,
-				authChallengeResponse: {
-					response: "ProvideCredentials",
-					username: options.httpProxyUsername,
-					password: options.httpProxyPassword
+			Fetch.addEventListener("authRequired", async ({ params }) => {
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Authenticating"]]);
 				}
-			}));
+				await Fetch.continueWithAuth({
+					requestId: params.requestId,
+					authChallengeResponse: {
+						response: "ProvideCredentials",
+						username: options.httpProxyUsername,
+						password: options.httpProxyPassword
+					}
+				});
+			});
 		}
 		let url = options.url;
 		let alternativeUrl = new URL(url);
@@ -134,12 +162,15 @@ async function getPageData(options) {
 		let httpInfo;
 		Fetch.addEventListener("requestPaused", async ({ params }) => {
 			const { requestId, request, resourceType, responseHeaders, responseStatusCode, responseStatusText } = params;
-			if (options.outputJson && !httpInfo && (request.url == url || request.url == alternativeUrl) && responseStatusCode !== undefined) {
+			if (resourceType == "Document" && (options.outputJson && !httpInfo && (request.url == url || request.url == alternativeUrl) && responseStatusCode !== undefined)) {
 				if (REDIRECT_STATUS_CODES.includes(responseStatusCode)) {
 					const redirect = responseHeaders.find(header => header.name.toLowerCase() == "location").value;
 					if (redirect) {
 						url = new URL(redirect, url).href;
 					}
+					if (options.debugMessagesFile) {
+						debugMessages.push([Date.now(), ["Redirecting", url]]);
+					}
 				} else {
 					httpInfo = {
 						request: {
@@ -156,18 +187,36 @@ async function getPageData(options) {
 						}
 					};
 				}
+			} else if (options.blockedURLPatterns && options.blockedURLPatterns.length) {
+				const blockedURL = options.blockedURLPatterns.find(blockedURL => new RegExp(blockedURL).test(request.url));
+				if (blockedURL) {
+					try {
+						if (options.debugMessagesFile) {
+							debugMessages.push([Date.now(), ["Blocking request", request.url]]);
+						}
+						await Fetch.failRequest({ requestId, errorReason: "Aborted" });
+					} catch {
+						// ignored
+					}
+				}
 			}
 			try {
 				await Fetch.continueRequest({ requestId });
-			} catch (error) {
+			} catch {
 				// ignored
 			}
 		});
 		if (options.httpHeaders && options.httpHeaders.length) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Setting HTTP headers", JSON.stringify(options.httpHeaders)]]);
+			}
 			await Network.setExtraHTTPHeaders({ headers: options.httpHeaders });
 		}
 		if (options.emulateMediaFeatures) {
 			for (const mediaFeature of options.emulateMediaFeatures) {
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Emulating media feature", mediaFeature.name, mediaFeature.value]]);
+				}
 				await Emulation.setEmulatedMedia({
 					media: mediaFeature.name,
 					features: mediaFeature.value.split(",").map(feature => feature.trim())
@@ -175,8 +224,12 @@ async function getPageData(options) {
 			}
 		}
 		if (options.browserCookies && options.browserCookies.length) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), ["Setting cookies", JSON.stringify(options.browserCookies)]]);
+			}
 			await Network.setCookies({ cookies: options.browserCookies });
 		}
+		await Browser.setDownloadBehavior({ behavior: "deny" });
 		await Page.addScriptToEvaluateOnNewDocument({
 			source: getHookScriptSource(),
 			runImmediately: true
@@ -187,7 +240,7 @@ async function getPageData(options) {
 			worldName: SINGLE_FILE_WORLD_NAME
 		});
 		const [contextId] = await Promise.all([
-			loadPage({ Page, Runtime }, options),
+			loadPage({ Page, Runtime }, options, debugMessages),
 			options.browserDebug ? waitForDebuggerReady({ Debugger }) : Promise.resolve()
 		]);
 		await Runtime.addBinding({ name: SET_PAGE_DATA_FUNCTION_NAME, executionContextId: contextId });
@@ -195,19 +248,22 @@ async function getPageData(options) {
 			await Runtime.addBinding({ name: CAPTURE_SCREENSHOT_FUNCTION_NAME, executionContextId: contextId });
 			Runtime.addEventListener("bindingCalled", async ({ params }) => {
 				if (params.name === CAPTURE_SCREENSHOT_FUNCTION_NAME) {
+					if (options.debugMessagesFile) {
+						debugMessages.push([Date.now(), ["Capturing screenshot"]]);
+					}
 					try {
 						let screenshotOptions = { captureBeyondViewport: true };
 						if (options.embedScreenshotOptions) {
 							try {
 								screenshotOptions = JSON.parse(options.embedScreenshotOptions);
-							} catch (error) {
+							} catch {
 								// ignored
 							}
 						}
 						screenshotOptions.format = "png";
 						const { data } = await Page.captureScreenshot(screenshotOptions);
 						await Runtime.evaluate({ expression: `globalThis.${SET_SCREENSHOT_FUNCTION_NAME}(${JSON.stringify(data)})`, contextId });
-					} catch (error) {
+					} catch {
 						await Runtime.evaluate({ expression: `globalThis.${SET_SCREENSHOT_FUNCTION_NAME}(${JSON.stringify("")})`, contextId });
 					}
 				}
@@ -217,18 +273,21 @@ async function getPageData(options) {
 			await Runtime.addBinding({ name: PRINT_TO_PDF_FUNCTION_NAME, executionContextId: contextId });
 			Runtime.addEventListener("bindingCalled", async ({ params }) => {
 				if (params.name === PRINT_TO_PDF_FUNCTION_NAME) {
+					if (options.debugMessagesFile) {
+						debugMessages.push([Date.now(), ["Printing to PDF", options.embedPdfOptions || ""]]);
+					}
 					let pdfOptions = {};
 					if (options.embedPdfOptions) {
 						try {
 							pdfOptions = JSON.parse(options.embedPdfOptions);
-						} catch (error) {
+						} catch {
 							// ignored
 						}
 					}
 					try {
 						const { data } = await Page.printToPDF(pdfOptions);
 						await Runtime.evaluate({ expression: `globalThis.${SET_PDF_FUNCTION_NAME}(${JSON.stringify(data)})`, contextId });
-					} catch (error) {
+					} catch {
 						await Runtime.evaluate({ expression: `globalThis.${SET_PDF_FUNCTION_NAME}(${JSON.stringify("")})`, contextId });
 					}
 				}
@@ -236,12 +295,15 @@ async function getPageData(options) {
 		}
 		const pageDataPromise = new Promise(resolve => {
 			let pageDataResponse = "";
-			Runtime.addEventListener("bindingCalled", async ({ params }) => {
+			Runtime.addEventListener("bindingCalled", ({ params }) => {
 				if (params.name === SET_PAGE_DATA_FUNCTION_NAME) {
 					const { payload } = params;
 					if (payload.length) {
 						pageDataResponse += payload;
 					} else {
+						if (options.debugMessagesFile) {
+							debugMessages.push([Date.now(), ["Setting page data"]]);
+						}
 						const result = JSON.parse(pageDataResponse);
 						if (result.content instanceof Array) {
 							result.content = new Uint8Array(result.content);
@@ -252,8 +314,14 @@ async function getPageData(options) {
 			});
 		});
 		if (options.browserWaitDelay) {
+			if (options.debugMessagesFile) {
+				debugMessages.push([Date.now(), [`Waiting ${options.browserWaitDelay} ms`]]);
+			}
 			await new Promise(resolve => setTimeout(resolve, options.browserWaitDelay));
 		}
+		if (options.debugMessagesFile) {
+			debugMessages.push([Date.now(), ["Capturing page"]]);
+		}
 		const { result } = await Runtime.evaluate({
 			expression: `(${getPageDataScriptSource.toString()})(${JSON.stringify(options)},${JSON.stringify([SET_SCREENSHOT_FUNCTION_NAME, SET_PDF_FUNCTION_NAME, SET_PAGE_DATA_FUNCTION_NAME, CAPTURE_SCREENSHOT_FUNCTION_NAME, PRINT_TO_PDF_FUNCTION_NAME])})`,
 			awaitPromise: true,
@@ -269,20 +337,42 @@ async function getPageData(options) {
 			await Console.disable();
 			pageData.consoleMessages = consoleMessages;
 		}
+		if (options.debugMessagesFile) {
+			pageData.debugMessages = debugMessages;
+			debugMessages.push([Date.now(), ["Returning page data"]]);
+		}
 		Object.assign(pageData, httpInfo);
 		return pageData;
 	} catch (error) {
 		if (error.code === LOAD_TIMEOUT_ERROR && options.browserWaitUntilFallback && options.browserWaitUntil) {
 			const browserWaitUntil = NETWORK_STATES[(NETWORK_STATES.indexOf(options.browserWaitUntil) + 1)];
 			if (browserWaitUntil) {
+				if (options.debugMessagesFile) {
+					debugMessages.push([Date.now(), ["Retrying with waitUntil", browserWaitUntil]]);
+				}
 				options.browserWaitUntil = browserWaitUntil;
+				await closeTarget();
 				return await getPageData(options);
 			}
 		}
+		if (options.consoleMessagesFile) {
+			error.consoleMessages = consoleMessages;
+		}
+		if (options.debugMessagesFile) {
+			error.debugMessages = debugMessages;
+		}
 		throw error;
 	} finally {
+		if (options.debugMessagesFile) {
+			debugMessages.push([Date.now(), ["Closing page"]]);
+		}
+		await closeTarget();
+	}
+
+	async function closeTarget() {
 		if (targetInfo && !options.browserDebug) {
 			await CDP.closeTarget(targetInfo.id);
+			targetInfo = null;
 		}
 	}
 }
@@ -328,14 +418,17 @@ function getPageDataScriptSource(options, [SET_SCREENSHOT_FUNCTION_NAME, SET_PDF
 	});
 }
 
-async function loadPage({ Page, Runtime }, options) {
+async function loadPage({ Page, Runtime }, options, debugMessages) {
 	await Runtime.enable();
 	await Page.enable();
 	const loadTimeoutAbortController = new AbortController();
 	const loadTimeoutAbortSignal = loadTimeoutAbortController.signal;
 	try {
+		if (options.debugMessagesFile) {
+			debugMessages.push([Date.now(), ["Loading page", options.url]]);
+		}
 		const [contextId] = await Promise.race([
-			Promise.all([getTopFrameContextId({ Page, Runtime }, options), Page.navigate({ url: options.url })]),
+			Promise.all([getTopFrameContextId({ Page, Runtime }, options, debugMessages), Page.navigate({ url: options.url })]),
 			waitForLoadTimeout(loadTimeoutAbortSignal, options.browserLoadMaxTime)
 		]);
 		return contextId;
@@ -348,13 +441,16 @@ async function loadPage({ Page, Runtime }, options) {
 	}
 }
 
-async function getTopFrameContextId({ Page, Runtime }, options) {
+async function getTopFrameContextId({ Page, Runtime }, options, debugMessages) {
 	const CONTEXT_CREATED_EVENT = "executionContextCreated";
 	const contextIds = [];
 	let topFrameId;
 	try {
 		Runtime.addEventListener(CONTEXT_CREATED_EVENT, onExecutionContextCreated);
 		await waitForPageReady({ Page }, options);
+		if (options.debugMessagesFile) {
+			debugMessages.push([Date.now(), ["Getting execution context"]]);
+		}
 		const contextId = await getContextId();
 		if (contextId === undefined) {
 			throw new Error("Execution context not found");
@@ -374,6 +470,7 @@ async function getTopFrameContextId({ Page, Runtime }, options) {
 	}
 
 	async function waitForPageReady({ Page }, options) {
+		let resolveCallbackTimeout;
 		await Page.setLifecycleEventsEnabled({ enabled: true });
 		try {
 			await new Promise((resolve, reject) => {
@@ -384,9 +481,23 @@ async function getTopFrameContextId({ Page, Runtime }, options) {
 
 				function onLifecycleEvent({ params }) {
 					const { frameId, name } = params;
-					if (frameId === topFrameId && name === options.browserWaitUntil) {
-						removeListeners();
-						resolve();
+					if (frameId === topFrameId) {
+						if (options.debugMessagesFile) {
+							debugMessages.push([Date.now(), ["Detecting lifecycle event", name]]);
+						}
+						if (name === options.browserWaitUntil || (resolveCallbackTimeout && NETWORK_STATES.indexOf(name) < NETWORK_STATES.indexOf(options.browserWaitUntil))) {
+							clearTimeout(resolveCallbackTimeout);
+							if (options.debugMessagesFile) {
+								debugMessages.push([Date.now(), [`Waiting ${options.browserWaitUntilDelay} ms`]]);
+							}
+							setTimeout(() => {
+								if (options.debugMessagesFile) {
+									debugMessages.push([Date.now(), ["Detecting page ready"]]);
+								}
+								removeListeners();
+								resolve();
+							}, options.browserWaitUntilDelay);
+						}
 					}
 				}
 
@@ -394,9 +505,13 @@ async function getTopFrameContextId({ Page, Runtime }, options) {
 					const { frame } = params;
 					if (!frame.parentId) {
 						if (frame.unreachableUrl) {
+							clearTimeout(resolveCallbackTimeout);
 							removeListeners();
 							reject(new Error("Unreachable URL: " + frame.unreachableUrl));
 						} else {
+							if (options.debugMessagesFile) {
+								debugMessages.push([Date.now(), ["Detecting top frame ID"]]);
+							}
 							topFrameId = frame.id;
 						}
 					}
@@ -433,7 +548,7 @@ async function getTopFrameContextId({ Page, Runtime }, options) {
 				contextId
 			});
 			return result.value === true;
-		} catch (error) {
+		} catch {
 			// ignored
 		}
 		return false;

package/lib/constants.js

@@ -51,7 +51,9 @@ const BROWSER_ARGS = [
 	"--enable-use-zoom-for-dsf=false",
 	"--no-sandbox",
 	"--no-startup-window",
-	"--bwsi"
+	"--bwsi",
+	"--mute-audio",
+	"--deny-permission-prompts"
 ];
 const BROWSER_PATHS = {
 	linux: [

package/lib/deno-polyfill.js

@@ -21,6 +21,8 @@
  *   Source.
  */
 
+// deno-lint-ignore-file no-node-globals
+
 /* global globalThis, URL, Deno, process */
 
 import * as path from "path";
@@ -244,7 +246,7 @@ async function cwd() {
 	}
 }
 
-async function dirname(filePath) {
+function dirname(filePath) {
 	return path.dirname(filePath);
 }