single-file-cli 2.3.0 → 2.3.1

package/build.sh

@@ -3,7 +3,7 @@
 mv package.json package.json.tmp
 mv deno.json deno.json.tmp
 mv deno.lock deno.lock.tmp
-deno install --vendor --quiet --minimum-dependency-age=0 "npm:single-file-core@1.5.95"
+deno install --vendor --quiet --minimum-dependency-age=0 "npm:single-file-core@1.5.98"
 mv package.json.tmp package.json
 mv deno.json.tmp deno.json
 mv deno.lock.tmp deno.lock

package/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.3.0",
+  "version": "2.3.1",
   "description": "SingleFile CLI",
   "exports": {
     ".": "./single-file-cli-api.js"

package/lib/cdp-client.js

@@ -102,7 +102,10 @@ function relaunchBrowser() {
 
 async function getPageData(options) {
 	const EMPTY_PAGE_URL = "about:blank";
-	const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {}, fetchAbortController: new AbortController() };
+	// compiled here so that an invalid pattern is reported before the page is
+	// loaded, instead of throwing for every request that is intercepted
+	const blockedURLPatterns = (options.blockedURLPatterns || []).map(pattern => new RegExp(pattern));
+	const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {}, blockedURLPatterns, fetchAbortController: new AbortController() };
 	let targetInfo, cdp;
 	try {
 		logData(["Loading page", EMPTY_PAGE_URL], pageContext);
@@ -205,25 +208,28 @@ async function setupBrowserWindow({ Browser }, targetId, { options, debugMessage
 	}
 }
 
-async function setupSecurity({ Security }, { options, debugMessages }) {
+async function setupSecurity({ Security }, { options, debugMessages }, sessionId) {
 	if (options.browserIgnoreHTTPSErrors !== undefined && options.browserIgnoreHTTPSErrors) {
 		logData(["Ignoring HTTPS errors"], { options, debugMessages });
-		await Security.setIgnoreCertificateErrors({ ignore: true });
+		await Security.setIgnoreCertificateErrors({ ignore: true }, sessionId);
 	}
 }
 
 async function setupDeviceEmulation({ Browser, Emulation, Runtime }, { options, debugMessages }) {
 	const needsDeviceMetrics = options.browserMobileEmulation || options.browserDeviceWidth ||
 		options.browserDeviceHeight || options.browserDeviceScaleFactor;
-	const needsUserAgent = options.browserMobileEmulation || options.platform || options.acceptLanguage;
 	if (needsDeviceMetrics) {
 		await setupDeviceMetrics({ Emulation, Runtime }, { options, debugMessages });
 	}
-	if (needsUserAgent) {
+	if (needsUserAgentOverride(options)) {
 		await setupUserAgent({ Browser, Emulation }, { options, debugMessages });
 	}
 }
 
+function needsUserAgentOverride(options) {
+	return Boolean(options.browserMobileEmulation || options.platform || options.acceptLanguage);
+}
+
 async function setupDeviceMetrics({ Emulation, Runtime }, { options, debugMessages }) {
 	const INNER_WIDTH_PROPERTY = "window.innerWidth";
 	const INNER_HEIGHT_PROPERTY = "window.innerHeight";
@@ -244,7 +250,7 @@ async function setupDeviceMetrics({ Emulation, Runtime }, { options, debugMessag
 	await Emulation.setDeviceMetricsOverride(deviceMetricsOptions);
 }
 
-async function setupUserAgent({ Browser, Emulation }, { options, debugMessages }) {
+async function setupUserAgent({ Browser, Emulation }, { options, debugMessages }, sessionId) {
 	const ANDROID_PLATFORM = "Android";
 	const { userAgent, product } = await Browser.getVersion();
 	const defaultMobileUA = `Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) ${product} Mobile Safari/537.36`;
@@ -258,22 +264,17 @@ async function setupUserAgent({ Browser, Emulation }, { options, debugMessages }
 		agentOptions.platform = options.platform || ANDROID_PLATFORM;
 	}
 	logData(["Emulating user agent", JSON.stringify(agentOptions)], { options, debugMessages });
-	await Emulation.setUserAgentOverride(agentOptions);
+	await Emulation.setUserAgentOverride(agentOptions, sessionId);
 }
 
-async function setupNetworkInterception({ Browser, Emulation, Fetch, Network }, { options, debugMessages, httpInfo }) {
-	const REQUEST_STAGE = "Request";
-	const RESPONSE_STAGE = "Response";
+async function setupNetworkInterception({ Browser, Emulation, Fetch, Network }, { options, debugMessages, httpInfo, blockedURLPatterns }) {
 	const DENY_BEHAVIOR = "deny";
-	const handleAuthRequests = Boolean(options.httpProxyUsername);
-	const patterns = handleAuthRequests ?
-		[{ requestStage: REQUEST_STAGE }, { requestStage: RESPONSE_STAGE }] :
-		[{ requestStage: RESPONSE_STAGE }];
+	const { handleAuthRequests, patterns } = getInterceptionOptions(options);
 	await Fetch.enable({ handleAuthRequests, patterns });
 	if (handleAuthRequests) {
 		setupProxyAuth({ Fetch }, { options, debugMessages });
 	}
-	setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo });
+	setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo, blockedURLPatterns });
 	if (options.httpHeaders) {
 		await setupHttpHeaders({ Network }, { options, debugMessages });
 	}
@@ -286,10 +287,20 @@ async function setupNetworkInterception({ Browser, Emulation, Fetch, Network },
 	await Browser.setDownloadBehavior({ behavior: DENY_BEHAVIOR });
 }
 
+function getInterceptionOptions(options) {
+	const REQUEST_STAGE = "Request";
+	const RESPONSE_STAGE = "Response";
+	const handleAuthRequests = Boolean(options.httpProxyUsername);
+	const patterns = handleAuthRequests ?
+		[{ requestStage: REQUEST_STAGE }, { requestStage: RESPONSE_STAGE }] :
+		[{ requestStage: RESPONSE_STAGE }];
+	return { handleAuthRequests, patterns };
+}
+
 function setupProxyAuth({ Fetch }, { options, debugMessages }) {
 	const AUTH_REQUIRED_EVENT_TYPE = "authRequired";
 	const PROVIDE_CREDENTIALS_RESPONSE = "ProvideCredentials";
-	Fetch.addEventListener(AUTH_REQUIRED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
+	Fetch.addEventListener(AUTH_REQUIRED_EVENT_TYPE, ignoringErrors(async ({ params, sessionId }) => {
 		logData(["Authenticating"], { options, debugMessages });
 		await Fetch.continueWithAuth({
 			requestId: params.requestId,
@@ -298,38 +309,37 @@ function setupProxyAuth({ Fetch }, { options, debugMessages }) {
 				username: options.httpProxyUsername,
 				password: options.httpProxyPassword
 			}
-		});
+		}, sessionId);
 	}, { options, debugMessages }));
 }
 
-function setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo }) {
+function setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo, blockedURLPatterns }) {
 	const REQUEST_PAUSED_EVENT_TYPE = "requestPaused";
 	const ABORTED_ERROR_REASON = "Aborted";
 	const urlState = { url: options.url, alternativeUrl: getAlternativeUrl(options.url) };
-	// compiled here so that an invalid pattern is reported before the page is
-	// loaded, instead of throwing for every request that is intercepted
-	const blockedURLPatterns = (options.blockedURLPatterns || []).map(pattern => new RegExp(pattern));
-	Fetch.addEventListener(REQUEST_PAUSED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
+	Fetch.addEventListener(REQUEST_PAUSED_EVENT_TYPE, ignoringErrors(async ({ params, sessionId }) => {
 		const { requestId, request } = params;
 		// the request is always resumed below, otherwise the page waits for it
 		// until the load timeout expires
 		let blocked = false;
 		try {
-			captureHttpInfo(params, urlState, { options, debugMessages, httpInfo });
+			if (!sessionId) {
+				captureHttpInfo(params, urlState, { options, debugMessages, httpInfo });
+			}
 			blocked = shouldBlockRequest(request.url);
 		} catch (error) {
 			logData(["Ignoring request interception error", error.message], { options, debugMessages });
 		}
 		if (blocked) {
 			try {
-				await Fetch.failRequest({ requestId, errorReason: ABORTED_ERROR_REASON });
+				await Fetch.failRequest({ requestId, errorReason: ABORTED_ERROR_REASON }, sessionId);
 				return;
 			} catch {
 				// ignored
 			}
 		}
 		try {
-			await Fetch.continueRequest({ requestId });
+			await Fetch.continueRequest({ requestId }, sessionId);
 		} catch {
 			// ignored
 		}
@@ -381,13 +391,13 @@ function captureHttpInfo(params, urlState, { options, debugMessages, httpInfo })
 	}
 }
 
-async function setupHttpHeaders({ Network }, { options, debugMessages }) {
+async function setupHttpHeaders({ Network }, { options, debugMessages }, sessionId) {
 	logData(["Setting HTTP headers", JSON.stringify(options.httpHeaders)], { options, debugMessages });
-	await Network.enable();
-	await Network.setExtraHTTPHeaders({ headers: options.httpHeaders });
+	await Network.enable({}, sessionId);
+	await Network.setExtraHTTPHeaders({ headers: options.httpHeaders }, sessionId);
 }
 
-async function setupMediaFeatures({ Emulation }, { options, debugMessages }) {
+async function setupMediaFeatures({ Emulation }, { options, debugMessages }, sessionId) {
 	const features = [];
 	for (const mediaFeature of options.emulateMediaFeatures) {
 		logData(["Emulating media feature", mediaFeature.name, mediaFeature.value], { options, debugMessages });
@@ -395,7 +405,7 @@ async function setupMediaFeatures({ Emulation }, { options, debugMessages }) {
 			features.push({ name: mediaFeature.name, value: value.trim() });
 		}
 	}
-	await Emulation.setEmulatedMedia({ features });
+	await Emulation.setEmulatedMedia({ features }, sessionId);
 }
 
 async function setupCookies({ Network }, { options, debugMessages }) {
@@ -403,16 +413,77 @@ async function setupCookies({ Network }, { options, debugMessages }) {
 	await Network.setCookies({ cookies: options.browserCookies });
 }
 
-async function setupScriptInjection({ Page }, { options }) {
+async function setupScriptInjection(cdp, { options, debugMessages }) {
+	const { Page } = cdp;
+	const scriptSource = await getScriptSource(options);
 	await Page.addScriptToEvaluateOnNewDocument({
 		source: getHookScriptSource(),
 		runImmediately: true
 	});
 	await Page.addScriptToEvaluateOnNewDocument({
-		source: await getScriptSource(options),
+		source: scriptSource,
 		runImmediately: true,
 		worldName: SINGLE_FILE_WORLD_NAME
 	});
+	await setupFrameScriptInjection(cdp, scriptSource, { options, debugMessages });
+}
+
+// out-of-process frames are separate targets, so the scripts registered above
+// do not reach them; they are attached paused, injected and resumed instead
+async function setupFrameScriptInjection(cdp, scriptSource, { options, debugMessages }) {
+	const { Page, Runtime, Target } = cdp;
+	const ATTACHED_TO_TARGET_EVENT_TYPE = "attachedToTarget";
+	const IFRAME_TARGET_TYPE = "iframe";
+	const autoAttachOptions = { autoAttach: true, waitForDebuggerOnStart: true, flatten: true };
+	Target.addEventListener(ATTACHED_TO_TARGET_EVENT_TYPE, ignoringErrors(async ({ params }) => {
+		const { sessionId, targetInfo, waitingForDebugger } = params;
+		try {
+			if (targetInfo.type === IFRAME_TARGET_TYPE) {
+				logData(["Injecting scripts into frame", targetInfo.url], { options, debugMessages });
+				// the scripts registered below only run when the Page domain is
+				// enabled on the session
+				await Page.enable({}, sessionId);
+				await Target.setAutoAttach(autoAttachOptions, sessionId);
+				await Page.addScriptToEvaluateOnNewDocument({
+					source: getHookScriptSource(),
+					runImmediately: true
+				}, sessionId);
+				await Page.addScriptToEvaluateOnNewDocument({
+					source: scriptSource,
+					runImmediately: true,
+					worldName: SINGLE_FILE_WORLD_NAME
+				}, sessionId);
+				await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME }, sessionId);
+				await setupFrameSession(cdp, sessionId, { options, debugMessages });
+			}
+		} finally {
+			// the target is always resumed, even when a setup command fails,
+			// otherwise it would stay paused until the load timeout expires
+			if (waitingForDebugger) {
+				await Runtime.runIfWaitingForDebugger({}, sessionId);
+			}
+		}
+	}, { options, debugMessages }));
+	await Target.setAutoAttach(autoAttachOptions);
+}
+
+async function setupFrameSession({ Browser, Emulation, Fetch, Network, Security }, sessionId, { options, debugMessages }) {
+	const { handleAuthRequests, patterns } = getInterceptionOptions(options);
+	if (handleAuthRequests || (options.blockedURLPatterns && options.blockedURLPatterns.length)) {
+		// the requestPaused and authRequired listeners are registered once on
+		// the connection and receive the events of every session
+		await Fetch.enable({ handleAuthRequests, patterns }, sessionId);
+	}
+	await setupSecurity({ Security }, { options, debugMessages }, sessionId);
+	if (options.httpHeaders) {
+		await setupHttpHeaders({ Network }, { options, debugMessages }, sessionId);
+	}
+	if (options.emulateMediaFeatures) {
+		await setupMediaFeatures({ Emulation }, { options, debugMessages }, sessionId);
+	}
+	if (needsUserAgentOverride(options)) {
+		await setupUserAgent({ Browser, Emulation }, { options, debugMessages }, sessionId);
+	}
 }
 
 async function getContextId({ Debugger, Page, Runtime }, { options, debugMessages }) {
@@ -578,7 +649,7 @@ function setupPageDataCapture({ Runtime }, { options, debugMessages }) {
 	});
 }
 
-async function setupBindings({ Page, Runtime }, { options, debugMessages, fetchAbortController }) {
+async function setupBindings({ Page, Runtime }, { options, debugMessages, blockedURLPatterns, fetchAbortController }) {
 	await Runtime.addBinding({ name: SET_PAGE_DATA_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
 	if (options.embedScreenshot && options.compressContent) {
 		await setupScreenshotCapture({ Page, Runtime }, { options, debugMessages });
@@ -587,9 +658,9 @@ async function setupBindings({ Page, Runtime }, { options, debugMessages, fetchA
 		await setupPdfCapture({ Page, Runtime }, { options, debugMessages });
 	}
 	await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
-	Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
+	Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params, sessionId }) => {
 		if (params.name === FETCH_FUNCTION_NAME) {
-			await handleFetchRequest({ Runtime }, params, { options, debugMessages, fetchAbortController });
+			await handleFetchRequest({ Runtime }, params, { options, debugMessages, blockedURLPatterns, fetchAbortController }, sessionId);
 		}
 	}, { options, debugMessages }));
 }
@@ -652,12 +723,23 @@ async function setupPdfCapture({ Page, Runtime }, { options, debugMessages }) {
 	}
 }
 
-async function handleFetchRequest({ Runtime }, params, { options, debugMessages, fetchAbortController }) {
+async function handleFetchRequest({ Runtime }, params, { options, debugMessages, blockedURLPatterns, fetchAbortController }, sessionId) {
+	const BLOCKED_URL_ERROR_MESSAGE = "Blocked URL";
 	const { executionContextId: contextId, payload } = params;
 	const { requestId, url, options: fetchOptions } = JSON.parse(payload);
 	logData(["Fetching URL", url], { options, debugMessages });
 	try {
-		const response = await fetch(url, Object.assign({}, fetchOptions, { signal: fetchAbortController.signal }));
+		// this fetch does not go through the browser, so the URL blocking and
+		// the extra HTTP headers must be applied here too
+		if (blockedURLPatterns.some(pattern => pattern.test(url))) {
+			logData(["Blocking request", url], { options, debugMessages });
+			throw new Error(BLOCKED_URL_ERROR_MESSAGE);
+		}
+		const headers = Object.assign({}, fetchOptions.headers, options.httpHeaders);
+		if (options.userAgent) {
+			headers["user-agent"] = options.userAgent;
+		}
+		const response = await fetch(url, Object.assign({}, fetchOptions, { headers, signal: fetchAbortController.signal }));
 		const arrayBuffer = await response.arrayBuffer();
 		const base64Data = arrayBufferToBase64(arrayBuffer);
 		const result = {
@@ -665,22 +747,22 @@ async function handleFetchRequest({ Runtime }, params, { options, debugMessages,
 			headers: Object.fromEntries(response.headers.entries()),
 			data: base64Data
 		};
-		await callBrowserFunction({ Runtime }, contextId, RESOLVE_FETCH_FUNCTION_NAME, [requestId, result]);
+		await callBrowserFunction({ Runtime }, contextId, RESOLVE_FETCH_FUNCTION_NAME, [requestId, result], sessionId);
 	} catch (error) {
 		const errorResult = {
 			error: error.message,
 			code: error.code
 		};
-		await callBrowserFunction({ Runtime }, contextId, REJECT_FETCH_FUNCTION_NAME, [requestId, errorResult]);
+		await callBrowserFunction({ Runtime }, contextId, REJECT_FETCH_FUNCTION_NAME, [requestId, errorResult], sessionId);
 	}
 }
 
-async function callBrowserFunction({ Runtime }, contextId, functionName, args) {
+async function callBrowserFunction({ Runtime }, contextId, functionName, args, sessionId) {
 	const serializedArgs = args.map(arg => JSON.stringify(arg)).join(", ");
 	await Runtime.evaluate({
 		expression: `globalThis.${functionName}(${serializedArgs})`,
 		contextId
-	});
+	}, sessionId);
 }
 
 async function capturePageData({ Runtime }, contextId, { options, debugMessages }) {