single-file-cli 2.3.0 → 2.4.0

package/lib/cdp-client.js

@@ -25,6 +25,7 @@
 
 import {
 	launchBrowser,
+	getBrowserOptions,
 	closeBrowser,
 	browserExited
 } from "./browser.js";
@@ -59,6 +60,7 @@ const SET_PDF_FUNCTION_NAME = "setPDF";
 const SET_PAGE_DATA_FUNCTION_NAME = "setPageData";
 const BINDING_CALLED_EVENT_TYPE = "bindingCalled";
 const LOCALHOST = "http://localhost:";
+const HEADLESS_USER_AGENT_TOKEN = "Headless";
 const BROWSER_EXITED_MAX_DELAY = 2000;
 
 let browserOptions, relaunchBrowserPromise;
@@ -73,17 +75,7 @@ async function initialize(singleFileOptions) {
 	if (singleFileOptions.browserServer) {
 		options.apiUrl = singleFileOptions.browserServer;
 	} else {
-		browserOptions = {};
-		browserOptions.args = singleFileOptions.browserArgs;
-		browserOptions.headless = singleFileOptions.browserHeadless;
-		browserOptions.executablePath = singleFileOptions.browserExecutablePath;
-		browserOptions.debug = singleFileOptions.browserDebug;
-		browserOptions.singleProcess = singleFileOptions.browserSingleProcess;
-		browserOptions.disableWebSecurity = singleFileOptions.browserDisableWebSecurity;
-		browserOptions.width = singleFileOptions.browserWidth;
-		browserOptions.height = singleFileOptions.browserHeight;
-		browserOptions.userAgent = singleFileOptions.userAgent;
-		browserOptions.httpProxyServer = singleFileOptions.httpProxyServer;
+		browserOptions = getBrowserOptions(singleFileOptions);
 		options.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
 	}
 }
@@ -102,7 +94,10 @@ function relaunchBrowser() {
 
 async function getPageData(options) {
 	const EMPTY_PAGE_URL = "about:blank";
-	const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {}, fetchAbortController: new AbortController() };
+	// compiled here so that an invalid pattern is reported before the page is
+	// loaded, instead of throwing for every request that is intercepted
+	const blockedURLPatterns = (options.blockedURLPatterns || []).map(pattern => new RegExp(pattern));
+	const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {}, blockedURLPatterns, fetchAbortController: new AbortController() };
 	let targetInfo, cdp;
 	try {
 		logData(["Loading page", EMPTY_PAGE_URL], pageContext);
@@ -205,23 +200,24 @@ async function setupBrowserWindow({ Browser }, targetId, { options, debugMessage
 	}
 }
 
-async function setupSecurity({ Security }, { options, debugMessages }) {
+async function setupSecurity({ Security }, { options, debugMessages }, sessionId) {
 	if (options.browserIgnoreHTTPSErrors !== undefined && options.browserIgnoreHTTPSErrors) {
 		logData(["Ignoring HTTPS errors"], { options, debugMessages });
-		await Security.setIgnoreCertificateErrors({ ignore: true });
+		await Security.setIgnoreCertificateErrors({ ignore: true }, sessionId);
 	}
 }
 
 async function setupDeviceEmulation({ Browser, Emulation, Runtime }, { options, debugMessages }) {
 	const needsDeviceMetrics = options.browserMobileEmulation || options.browserDeviceWidth ||
 		options.browserDeviceHeight || options.browserDeviceScaleFactor;
-	const needsUserAgent = options.browserMobileEmulation || options.platform || options.acceptLanguage;
 	if (needsDeviceMetrics) {
 		await setupDeviceMetrics({ Emulation, Runtime }, { options, debugMessages });
 	}
-	if (needsUserAgent) {
-		await setupUserAgent({ Browser, Emulation }, { options, debugMessages });
-	}
+	await setupUserAgent({ Browser, Emulation }, { options, debugMessages });
+}
+
+function needsUserAgentOverride(options) {
+	return Boolean(options.browserMobileEmulation || options.platform || options.acceptLanguage);
 }
 
 async function setupDeviceMetrics({ Emulation, Runtime }, { options, debugMessages }) {
@@ -244,12 +240,12 @@ async function setupDeviceMetrics({ Emulation, Runtime }, { options, debugMessag
 	await Emulation.setDeviceMetricsOverride(deviceMetricsOptions);
 }
 
-async function setupUserAgent({ Browser, Emulation }, { options, debugMessages }) {
+async function setupUserAgent({ Browser, Emulation }, { options, debugMessages }, sessionId) {
 	const ANDROID_PLATFORM = "Android";
 	const { userAgent, product } = await Browser.getVersion();
-	const defaultMobileUA = `Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) ${product} Mobile Safari/537.36`;
+	const defaultMobileUA = `Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) ${removeHeadlessToken(product)} Mobile Safari/537.36`;
 	const agentOptions = {
-		userAgent: options.userAgent || (options.browserMobileEmulation ? defaultMobileUA : userAgent)
+		userAgent: options.userAgent || (options.browserMobileEmulation ? defaultMobileUA : removeHeadlessToken(userAgent))
 	};
 	if (options.acceptLanguage) {
 		agentOptions.acceptLanguage = options.acceptLanguage;
@@ -257,23 +253,24 @@ async function setupUserAgent({ Browser, Emulation }, { options, debugMessages }
 	if (options.platform || options.browserMobileEmulation) {
 		agentOptions.platform = options.platform || ANDROID_PLATFORM;
 	}
-	logData(["Emulating user agent", JSON.stringify(agentOptions)], { options, debugMessages });
-	await Emulation.setUserAgentOverride(agentOptions);
+	if (needsUserAgentOverride(options) || agentOptions.userAgent !== userAgent) {
+		logData(["Emulating user agent", JSON.stringify(agentOptions)], { options, debugMessages });
+		await Emulation.setUserAgentOverride(agentOptions, sessionId);
+	}
 }
 
-async function setupNetworkInterception({ Browser, Emulation, Fetch, Network }, { options, debugMessages, httpInfo }) {
-	const REQUEST_STAGE = "Request";
-	const RESPONSE_STAGE = "Response";
+function removeHeadlessToken(userAgent) {
+	return userAgent.replace(HEADLESS_USER_AGENT_TOKEN, "");
+}
+
+async function setupNetworkInterception({ Browser, Emulation, Fetch, Network }, { options, debugMessages, httpInfo, blockedURLPatterns }) {
 	const DENY_BEHAVIOR = "deny";
-	const handleAuthRequests = Boolean(options.httpProxyUsername);
-	const patterns = handleAuthRequests ?
-		[{ requestStage: REQUEST_STAGE }, { requestStage: RESPONSE_STAGE }] :
-		[{ requestStage: RESPONSE_STAGE }];
+	const { handleAuthRequests, patterns } = getInterceptionOptions(options);
 	await Fetch.enable({ handleAuthRequests, patterns });
 	if (handleAuthRequests) {
 		setupProxyAuth({ Fetch }, { options, debugMessages });
 	}
-	setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo });
+	setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo, blockedURLPatterns });
 	if (options.httpHeaders) {
 		await setupHttpHeaders({ Network }, { options, debugMessages });
 	}
@@ -286,10 +283,20 @@ async function setupNetworkInterception({ Browser, Emulation, Fetch, Network },
 	await Browser.setDownloadBehavior({ behavior: DENY_BEHAVIOR });
 }
 
+function getInterceptionOptions(options) {
+	const REQUEST_STAGE = "Request";
+	const RESPONSE_STAGE = "Response";
+	const handleAuthRequests = Boolean(options.httpProxyUsername);
+	const patterns = handleAuthRequests ?
+		[{ requestStage: REQUEST_STAGE }, { requestStage: RESPONSE_STAGE }] :
+		[{ requestStage: RESPONSE_STAGE }];
+	return { handleAuthRequests, patterns };
+}
+
 function setupProxyAuth({ Fetch }, { options, debugMessages }) {
 	const AUTH_REQUIRED_EVENT_TYPE = "authRequired";
 	const PROVIDE_CREDENTIALS_RESPONSE = "ProvideCredentials";
-	Fetch.addEventListener(AUTH_REQUIRED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
+	Fetch.addEventListener(AUTH_REQUIRED_EVENT_TYPE, ignoringErrors(async ({ params, sessionId }) => {
 		logData(["Authenticating"], { options, debugMessages });
 		await Fetch.continueWithAuth({
 			requestId: params.requestId,
@@ -298,38 +305,37 @@ function setupProxyAuth({ Fetch }, { options, debugMessages }) {
 				username: options.httpProxyUsername,
 				password: options.httpProxyPassword
 			}
-		});
+		}, sessionId);
 	}, { options, debugMessages }));
 }
 
-function setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo }) {
+function setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo, blockedURLPatterns }) {
 	const REQUEST_PAUSED_EVENT_TYPE = "requestPaused";
 	const ABORTED_ERROR_REASON = "Aborted";
 	const urlState = { url: options.url, alternativeUrl: getAlternativeUrl(options.url) };
-	// compiled here so that an invalid pattern is reported before the page is
-	// loaded, instead of throwing for every request that is intercepted
-	const blockedURLPatterns = (options.blockedURLPatterns || []).map(pattern => new RegExp(pattern));
-	Fetch.addEventListener(REQUEST_PAUSED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
+	Fetch.addEventListener(REQUEST_PAUSED_EVENT_TYPE, ignoringErrors(async ({ params, sessionId }) => {
 		const { requestId, request } = params;
 		// the request is always resumed below, otherwise the page waits for it
 		// until the load timeout expires
 		let blocked = false;
 		try {
-			captureHttpInfo(params, urlState, { options, debugMessages, httpInfo });
+			if (!sessionId) {
+				captureHttpInfo(params, urlState, { options, debugMessages, httpInfo });
+			}
 			blocked = shouldBlockRequest(request.url);
 		} catch (error) {
 			logData(["Ignoring request interception error", error.message], { options, debugMessages });
 		}
 		if (blocked) {
 			try {
-				await Fetch.failRequest({ requestId, errorReason: ABORTED_ERROR_REASON });
+				await Fetch.failRequest({ requestId, errorReason: ABORTED_ERROR_REASON }, sessionId);
 				return;
 			} catch {
 				// ignored
 			}
 		}
 		try {
-			await Fetch.continueRequest({ requestId });
+			await Fetch.continueRequest({ requestId }, sessionId);
 		} catch {
 			// ignored
 		}
@@ -381,13 +387,13 @@ function captureHttpInfo(params, urlState, { options, debugMessages, httpInfo })
 	}
 }
 
-async function setupHttpHeaders({ Network }, { options, debugMessages }) {
+async function setupHttpHeaders({ Network }, { options, debugMessages }, sessionId) {
 	logData(["Setting HTTP headers", JSON.stringify(options.httpHeaders)], { options, debugMessages });
-	await Network.enable();
-	await Network.setExtraHTTPHeaders({ headers: options.httpHeaders });
+	await Network.enable({}, sessionId);
+	await Network.setExtraHTTPHeaders({ headers: options.httpHeaders }, sessionId);
 }
 
-async function setupMediaFeatures({ Emulation }, { options, debugMessages }) {
+async function setupMediaFeatures({ Emulation }, { options, debugMessages }, sessionId) {
 	const features = [];
 	for (const mediaFeature of options.emulateMediaFeatures) {
 		logData(["Emulating media feature", mediaFeature.name, mediaFeature.value], { options, debugMessages });
@@ -395,7 +401,7 @@ async function setupMediaFeatures({ Emulation }, { options, debugMessages }) {
 			features.push({ name: mediaFeature.name, value: value.trim() });
 		}
 	}
-	await Emulation.setEmulatedMedia({ features });
+	await Emulation.setEmulatedMedia({ features }, sessionId);
 }
 
 async function setupCookies({ Network }, { options, debugMessages }) {
@@ -403,16 +409,75 @@ async function setupCookies({ Network }, { options, debugMessages }) {
 	await Network.setCookies({ cookies: options.browserCookies });
 }
 
-async function setupScriptInjection({ Page }, { options }) {
+async function setupScriptInjection(cdp, { options, debugMessages }) {
+	const { Page } = cdp;
+	const scriptSource = await getScriptSource(options);
 	await Page.addScriptToEvaluateOnNewDocument({
 		source: getHookScriptSource(),
 		runImmediately: true
 	});
 	await Page.addScriptToEvaluateOnNewDocument({
-		source: await getScriptSource(options),
+		source: scriptSource,
 		runImmediately: true,
 		worldName: SINGLE_FILE_WORLD_NAME
 	});
+	await setupFrameScriptInjection(cdp, scriptSource, { options, debugMessages });
+}
+
+// out-of-process frames are separate targets, so the scripts registered above
+// do not reach them; they are attached paused, injected and resumed instead
+async function setupFrameScriptInjection(cdp, scriptSource, { options, debugMessages }) {
+	const { Page, Runtime, Target } = cdp;
+	const ATTACHED_TO_TARGET_EVENT_TYPE = "attachedToTarget";
+	const IFRAME_TARGET_TYPE = "iframe";
+	const autoAttachOptions = { autoAttach: true, waitForDebuggerOnStart: true, flatten: true };
+	Target.addEventListener(ATTACHED_TO_TARGET_EVENT_TYPE, ignoringErrors(async ({ params }) => {
+		const { sessionId, targetInfo, waitingForDebugger } = params;
+		try {
+			if (targetInfo.type === IFRAME_TARGET_TYPE) {
+				logData(["Injecting scripts into frame", targetInfo.url], { options, debugMessages });
+				// the scripts registered below only run when the Page domain is
+				// enabled on the session
+				await Page.enable({}, sessionId);
+				await Target.setAutoAttach(autoAttachOptions, sessionId);
+				await Page.addScriptToEvaluateOnNewDocument({
+					source: getHookScriptSource(),
+					runImmediately: true
+				}, sessionId);
+				await Page.addScriptToEvaluateOnNewDocument({
+					source: scriptSource,
+					runImmediately: true,
+					worldName: SINGLE_FILE_WORLD_NAME
+				}, sessionId);
+				await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME }, sessionId);
+				await setupFrameSession(cdp, sessionId, { options, debugMessages });
+			}
+		} finally {
+			// the target is always resumed, even when a setup command fails,
+			// otherwise it would stay paused until the load timeout expires
+			if (waitingForDebugger) {
+				await Runtime.runIfWaitingForDebugger({}, sessionId);
+			}
+		}
+	}, { options, debugMessages }));
+	await Target.setAutoAttach(autoAttachOptions);
+}
+
+async function setupFrameSession({ Browser, Emulation, Fetch, Network, Security }, sessionId, { options, debugMessages }) {
+	const { handleAuthRequests, patterns } = getInterceptionOptions(options);
+	if (handleAuthRequests || (options.blockedURLPatterns && options.blockedURLPatterns.length)) {
+		// the requestPaused and authRequired listeners are registered once on
+		// the connection and receive the events of every session
+		await Fetch.enable({ handleAuthRequests, patterns }, sessionId);
+	}
+	await setupSecurity({ Security }, { options, debugMessages }, sessionId);
+	if (options.httpHeaders) {
+		await setupHttpHeaders({ Network }, { options, debugMessages }, sessionId);
+	}
+	if (options.emulateMediaFeatures) {
+		await setupMediaFeatures({ Emulation }, { options, debugMessages }, sessionId);
+	}
+	await setupUserAgent({ Browser, Emulation }, { options, debugMessages }, sessionId);
 }
 
 async function getContextId({ Debugger, Page, Runtime }, { options, debugMessages }) {
@@ -578,7 +643,7 @@ function setupPageDataCapture({ Runtime }, { options, debugMessages }) {
 	});
 }
 
-async function setupBindings({ Page, Runtime }, { options, debugMessages, fetchAbortController }) {
+async function setupBindings({ Page, Runtime }, { options, debugMessages, blockedURLPatterns, fetchAbortController }) {
 	await Runtime.addBinding({ name: SET_PAGE_DATA_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
 	if (options.embedScreenshot && options.compressContent) {
 		await setupScreenshotCapture({ Page, Runtime }, { options, debugMessages });
@@ -587,9 +652,9 @@ async function setupBindings({ Page, Runtime }, { options, debugMessages, fetchA
 		await setupPdfCapture({ Page, Runtime }, { options, debugMessages });
 	}
 	await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
-	Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
+	Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params, sessionId }) => {
 		if (params.name === FETCH_FUNCTION_NAME) {
-			await handleFetchRequest({ Runtime }, params, { options, debugMessages, fetchAbortController });
+			await handleFetchRequest({ Runtime }, params, { options, debugMessages, blockedURLPatterns, fetchAbortController }, sessionId);
 		}
 	}, { options, debugMessages }));
 }
@@ -652,12 +717,23 @@ async function setupPdfCapture({ Page, Runtime }, { options, debugMessages }) {
 	}
 }
 
-async function handleFetchRequest({ Runtime }, params, { options, debugMessages, fetchAbortController }) {
+async function handleFetchRequest({ Runtime }, params, { options, debugMessages, blockedURLPatterns, fetchAbortController }, sessionId) {
+	const BLOCKED_URL_ERROR_MESSAGE = "Blocked URL";
 	const { executionContextId: contextId, payload } = params;
 	const { requestId, url, options: fetchOptions } = JSON.parse(payload);
 	logData(["Fetching URL", url], { options, debugMessages });
 	try {
-		const response = await fetch(url, Object.assign({}, fetchOptions, { signal: fetchAbortController.signal }));
+		// this fetch does not go through the browser, so the URL blocking and
+		// the extra HTTP headers must be applied here too
+		if (blockedURLPatterns.some(pattern => pattern.test(url))) {
+			logData(["Blocking request", url], { options, debugMessages });
+			throw new Error(BLOCKED_URL_ERROR_MESSAGE);
+		}
+		const headers = Object.assign({}, fetchOptions.headers, options.httpHeaders);
+		if (options.userAgent) {
+			headers["user-agent"] = options.userAgent;
+		}
+		const response = await fetch(url, Object.assign({}, fetchOptions, { headers, signal: fetchAbortController.signal }));
 		const arrayBuffer = await response.arrayBuffer();
 		const base64Data = arrayBufferToBase64(arrayBuffer);
 		const result = {
@@ -665,22 +741,22 @@ async function handleFetchRequest({ Runtime }, params, { options, debugMessages,
 			headers: Object.fromEntries(response.headers.entries()),
 			data: base64Data
 		};
-		await callBrowserFunction({ Runtime }, contextId, RESOLVE_FETCH_FUNCTION_NAME, [requestId, result]);
+		await callBrowserFunction({ Runtime }, contextId, RESOLVE_FETCH_FUNCTION_NAME, [requestId, result], sessionId);
 	} catch (error) {
 		const errorResult = {
 			error: error.message,
 			code: error.code
 		};
-		await callBrowserFunction({ Runtime }, contextId, REJECT_FETCH_FUNCTION_NAME, [requestId, errorResult]);
+		await callBrowserFunction({ Runtime }, contextId, REJECT_FETCH_FUNCTION_NAME, [requestId, errorResult], sessionId);
 	}
 }
 
-async function callBrowserFunction({ Runtime }, contextId, functionName, args) {
+async function callBrowserFunction({ Runtime }, contextId, functionName, args, sessionId) {
 	const serializedArgs = args.map(arg => JSON.stringify(arg)).join(", ");
 	await Runtime.evaluate({
 		expression: `globalThis.${functionName}(${serializedArgs})`,
 		contextId
-	});
+	}, sessionId);
 }
 
 async function capturePageData({ Runtime }, contextId, { options, debugMessages }) {

package/lib/constants.js

@@ -43,7 +43,6 @@ const BROWSER_ARGS = [
 	"--disable-renderer-backgrounding",
 	"--force-color-profile=srgb",
 	"--no-first-run",
-	"--enable-automation",
 	"--password-store=basic",
 	"--use-mock-keychain",
 	"--no-service-autorun",

package/lib/deno-polyfill.js

@@ -101,8 +101,10 @@ const DenoAPI = {
 	args,
 	readFile,
 	readTextFile,
+	readDir,
 	writeTextFile,
 	writeFile,
+	copyFile,
 	mkdir,
 	makeTempDir,
 	stat,
@@ -118,6 +120,7 @@ const DenoAPI = {
 
 const pathAPI = {
 	dirname,
+	join,
 	toFileUrl,
 	fromFileUrl,
 	SEPARATOR: DENO_RUNTIME_DETECTED ? path.SEPARATOR : path.sep
@@ -154,6 +157,41 @@ async function readTextFile(path) {
 	}
 }
 
+async function readDir(path) {
+	if (DENO_RUNTIME_DETECTED) {
+		const entries = [];
+		try {
+			for await (const entry of Deno.readDir(path)) {
+				entries.push({ name: entry.name, isDirectory: entry.isDirectory, isSymlink: entry.isSymlink });
+			}
+		} catch (error) {
+			throw mapNotFoundError(error);
+		}
+		return entries;
+	} else {
+		const fsPromise = await import(NODE_MODULES["fs"]);
+		try {
+			const entries = await fsPromise.readdir(path, { withFileTypes: true });
+			return entries.map(entry => ({ name: entry.name, isDirectory: entry.isDirectory(), isSymlink: entry.isSymbolicLink() }));
+		} catch (error) {
+			throw mapNotFoundError(error);
+		}
+	}
+}
+
+async function copyFile(sourcePath, destinationPath) {
+	if (DENO_RUNTIME_DETECTED) {
+		return Deno.copyFile(sourcePath, destinationPath);
+	} else {
+		const fsPromise = await import(NODE_MODULES["fs"]);
+		return fsPromise.copyFile(sourcePath, destinationPath);
+	}
+}
+
+function mapNotFoundError(error) {
+	return error.code == "ENOENT" || error.code == "ENOTDIR" ? new errors.NotFound(error.message) : error;
+}
+
 async function writeTextFile(path, data, options = {}) {
 	if (DENO_RUNTIME_DETECTED) {
 		return Deno.writeTextFile(path, data, options);
@@ -259,6 +297,10 @@ function dirname(filePath) {
 	return path.dirname(filePath);
 }
 
+function join(...filePaths) {
+	return path.join(...filePaths);
+}
+
 async function toFileUrl(filePath) {
 	if (DENO_RUNTIME_DETECTED) {
 		return path.toFileUrl(filePath);