npm - single-file-cli - Versions diffs - 2.2.2 → 2.3.1 - Mend

single-file-cli 2.2.2 → 2.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (19) hide show

package/Dockerfile +1 -0
package/README.MD +6 -2
package/build.sh +1 -1
package/deno.json +1 -1
package/lib/archive-packager.js +3 -0
package/lib/cdp-client-util.js +4 -1
package/lib/cdp-client.js +144 -58
package/lib/single-file-archive.js +6 -6
package/lib/single-file-bundle.js +1 -1
package/lib/version.js +1 -1
package/options.js +9 -4
package/package.json +1 -1
package/single-file-cli-api.js +17 -2
package/single-file-launcher.js +4 -1
package/test/e2e/errors-file.test.js +11 -5
package/test/e2e/oopif.test.js +86 -0
package/test/e2e/urls-file.test.js +31 -0
package/test/unit/archive-packager.test.js +15 -2
package/test/unit/options.test.js +3 -0

package/Dockerfile CHANGED Viewed

@@ -14,5 +14,6 @@ ENTRYPOINT [ \
     "npx", \
     "single-file", \
     "--browser-executable-path", "/usr/bin/chromium-browser", \
+    "--browser-single-process=false", \
     "--output-directory", "./out/", \
     "--dump-content" ]

package/README.MD CHANGED Viewed

@@ -143,12 +143,16 @@ Make sure Chrome or a Chromium-based browser is installed in the default folder.
   single-file https://www.wikipedia.org --crawl-links=true --crawl-inner-links-only=false --crawl-external-links-max-depth=1 --crawl-rewrite-rule="^.*wikipedia.*$"
   ```
-  - Save https://www.wikipedia.org and its internal links into a single self-extracting ZIP file
+  - Save http://info.cern.ch (i.e. the first website) and its internal links into a single self-extracting ZIP file, and mark the links to pages missing from the archive
   ```sh
-  single-file https://www.wikipedia.org wikipedia.html --crawl-links=true --crawl-save-archive=true --compress-content=true
+  single-file http://info.cern.ch first-website.html --crawl-links=true --crawl-max-depth=2 --crawl-save-archive=true --crawl-save-archive-mark-unarchived-links=true --compress-content=true
   ```
+- Exit code
+  The exit code is 0 when all the pages have been saved, 1 when at least one capture failed (the other pages of the batch or the crawl are still saved), and 255 when an error prevented the process from running.
 ## Compile executables
  - Compile executables into `/dist`

package/build.sh CHANGED Viewed

@@ -3,7 +3,7 @@
 mv package.json package.json.tmp
 mv deno.json deno.json.tmp
 mv deno.lock deno.lock.tmp
-deno install --vendor --quiet --minimum-dependency-age=0 "npm:single-file-core@1.5.92"
+deno install --vendor --quiet --minimum-dependency-age=0 "npm:single-file-core@1.5.98"
 mv package.json.tmp package.json
 mv deno.json.tmp deno.json
 mv deno.lock.tmp deno.lock

package/deno.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.2.2",
+  "version": "2.3.1",
   "description": "SingleFile CLI",
   "exports": {
     ".": "./single-file-cli-api.js"

package/lib/archive-packager.js CHANGED Viewed

@@ -59,6 +59,9 @@ async function createPagesArchive(pages, options) {
 	if (options.markUnarchivedLinks) {
 		manifest.markUnarchivedLinks = true;
 	}
+	if (options.pageTransitions && options.pageTransitions != "auto") {
+		manifest.pageTransitions = options.pageTransitions;
+	}
 	const pageData = {
 		doctype: "<!DOCTYPE html>",
 		content: "",

package/lib/cdp-client-util.js CHANGED Viewed

@@ -74,10 +74,13 @@ function waitForTimeout(abortSignal, maxDelay, errorMessage, errorCode) {
 			reject(error);
 		}, maxDelay);
+		// the promise must stay pending on abort: the abort fires after the
+		// Promise.race around this timeout has settled, and settling here would
+		// either make the race winner destructure undefined or raise an
+		// unhandled rejection
 		function onAbort() {
 			abortSignal.removeEventListener(ABORT_EVENT, onAbort);
 			clearTimeout(timeoutId);
-			resolve();
 		}
 	});
 }

package/lib/cdp-client.js CHANGED Viewed

@@ -102,7 +102,10 @@ function relaunchBrowser() {
 async function getPageData(options) {
 	const EMPTY_PAGE_URL = "about:blank";
-	const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {}, fetchAbortController: new AbortController() };
+	// compiled here so that an invalid pattern is reported before the page is
+	// loaded, instead of throwing for every request that is intercepted
+	const blockedURLPatterns = (options.blockedURLPatterns || []).map(pattern => new RegExp(pattern));
+	const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {}, blockedURLPatterns, fetchAbortController: new AbortController() };
 	let targetInfo, cdp;
 	try {
 		logData(["Loading page", EMPTY_PAGE_URL], pageContext);
@@ -115,8 +118,8 @@ async function getPageData(options) {
 		await setupNetworkInterception(cdp, pageContext);
 		await setupScriptInjection(cdp, pageContext);
 		const contextId = await getContextId(cdp, pageContext);
-		const pageDataPromise = setupPageDataCapture(cdp, contextId, pageContext);
-		await setupBindings(cdp, contextId, pageContext);
+		const pageDataPromise = setupPageDataCapture(cdp, pageContext);
+		await setupBindings(cdp, pageContext);
 		await capturePageData(cdp, contextId, pageContext);
 		await disableCdpDomains(cdp, pageContext);
 		return await finalizePageData(pageDataPromise, pageContext);
@@ -205,25 +208,28 @@ async function setupBrowserWindow({ Browser }, targetId, { options, debugMessage
 	}
 }
-async function setupSecurity({ Security }, { options, debugMessages }) {
+async function setupSecurity({ Security }, { options, debugMessages }, sessionId) {
 	if (options.browserIgnoreHTTPSErrors !== undefined && options.browserIgnoreHTTPSErrors) {
 		logData(["Ignoring HTTPS errors"], { options, debugMessages });
-		await Security.setIgnoreCertificateErrors({ ignore: true });
+		await Security.setIgnoreCertificateErrors({ ignore: true }, sessionId);
 	}
 }
 async function setupDeviceEmulation({ Browser, Emulation, Runtime }, { options, debugMessages }) {
 	const needsDeviceMetrics = options.browserMobileEmulation || options.browserDeviceWidth ||
 		options.browserDeviceHeight || options.browserDeviceScaleFactor;
-	const needsUserAgent = options.browserMobileEmulation || options.platform || options.acceptLanguage;
 	if (needsDeviceMetrics) {
 		await setupDeviceMetrics({ Emulation, Runtime }, { options, debugMessages });
 	}
-	if (needsUserAgent) {
+	if (needsUserAgentOverride(options)) {
 		await setupUserAgent({ Browser, Emulation }, { options, debugMessages });
 	}
 }
+function needsUserAgentOverride(options) {
+	return Boolean(options.browserMobileEmulation || options.platform || options.acceptLanguage);
+}
 async function setupDeviceMetrics({ Emulation, Runtime }, { options, debugMessages }) {
 	const INNER_WIDTH_PROPERTY = "window.innerWidth";
 	const INNER_HEIGHT_PROPERTY = "window.innerHeight";
@@ -244,7 +250,7 @@ async function setupDeviceMetrics({ Emulation, Runtime }, { options, debugMessag
 	await Emulation.setDeviceMetricsOverride(deviceMetricsOptions);
 }
-async function setupUserAgent({ Browser, Emulation }, { options, debugMessages }) {
+async function setupUserAgent({ Browser, Emulation }, { options, debugMessages }, sessionId) {
 	const ANDROID_PLATFORM = "Android";
 	const { userAgent, product } = await Browser.getVersion();
 	const defaultMobileUA = `Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) ${product} Mobile Safari/537.36`;
@@ -258,22 +264,17 @@ async function setupUserAgent({ Browser, Emulation }, { options, debugMessages }
 		agentOptions.platform = options.platform || ANDROID_PLATFORM;
 	}
 	logData(["Emulating user agent", JSON.stringify(agentOptions)], { options, debugMessages });
-	await Emulation.setUserAgentOverride(agentOptions);
+	await Emulation.setUserAgentOverride(agentOptions, sessionId);
 }
-async function setupNetworkInterception({ Browser, Emulation, Fetch, Network }, { options, debugMessages, httpInfo }) {
-	const REQUEST_STAGE = "Request";
-	const RESPONSE_STAGE = "Response";
+async function setupNetworkInterception({ Browser, Emulation, Fetch, Network }, { options, debugMessages, httpInfo, blockedURLPatterns }) {
 	const DENY_BEHAVIOR = "deny";
-	const handleAuthRequests = Boolean(options.httpProxyUsername);
-	const patterns = handleAuthRequests ?
-		[{ requestStage: REQUEST_STAGE }, { requestStage: RESPONSE_STAGE }] :
-		[{ requestStage: RESPONSE_STAGE }];
+	const { handleAuthRequests, patterns } = getInterceptionOptions(options);
 	await Fetch.enable({ handleAuthRequests, patterns });
 	if (handleAuthRequests) {
 		setupProxyAuth({ Fetch }, { options, debugMessages });
 	}
-	setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo });
+	setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo, blockedURLPatterns });
 	if (options.httpHeaders) {
 		await setupHttpHeaders({ Network }, { options, debugMessages });
 	}
@@ -286,10 +287,20 @@ async function setupNetworkInterception({ Browser, Emulation, Fetch, Network },
 	await Browser.setDownloadBehavior({ behavior: DENY_BEHAVIOR });
 }
+function getInterceptionOptions(options) {
+	const REQUEST_STAGE = "Request";
+	const RESPONSE_STAGE = "Response";
+	const handleAuthRequests = Boolean(options.httpProxyUsername);
+	const patterns = handleAuthRequests ?
+		[{ requestStage: REQUEST_STAGE }, { requestStage: RESPONSE_STAGE }] :
+		[{ requestStage: RESPONSE_STAGE }];
+	return { handleAuthRequests, patterns };
+}
 function setupProxyAuth({ Fetch }, { options, debugMessages }) {
 	const AUTH_REQUIRED_EVENT_TYPE = "authRequired";
 	const PROVIDE_CREDENTIALS_RESPONSE = "ProvideCredentials";
-	Fetch.addEventListener(AUTH_REQUIRED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
+	Fetch.addEventListener(AUTH_REQUIRED_EVENT_TYPE, ignoringErrors(async ({ params, sessionId }) => {
 		logData(["Authenticating"], { options, debugMessages });
 		await Fetch.continueWithAuth({
 			requestId: params.requestId,
@@ -298,38 +309,37 @@ function setupProxyAuth({ Fetch }, { options, debugMessages }) {
 				username: options.httpProxyUsername,
 				password: options.httpProxyPassword
 			}
-		});
+		}, sessionId);
 	}, { options, debugMessages }));
 }
-function setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo }) {
+function setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo, blockedURLPatterns }) {
 	const REQUEST_PAUSED_EVENT_TYPE = "requestPaused";
 	const ABORTED_ERROR_REASON = "Aborted";
 	const urlState = { url: options.url, alternativeUrl: getAlternativeUrl(options.url) };
-	// compiled here so that an invalid pattern is reported before the page is
-	// loaded, instead of throwing for every request that is intercepted
-	const blockedURLPatterns = (options.blockedURLPatterns || []).map(pattern => new RegExp(pattern));
-	Fetch.addEventListener(REQUEST_PAUSED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
+	Fetch.addEventListener(REQUEST_PAUSED_EVENT_TYPE, ignoringErrors(async ({ params, sessionId }) => {
 		const { requestId, request } = params;
 		// the request is always resumed below, otherwise the page waits for it
 		// until the load timeout expires
 		let blocked = false;
 		try {
-			captureHttpInfo(params, urlState, { options, debugMessages, httpInfo });
+			if (!sessionId) {
+				captureHttpInfo(params, urlState, { options, debugMessages, httpInfo });
+			}
 			blocked = shouldBlockRequest(request.url);
 		} catch (error) {
 			logData(["Ignoring request interception error", error.message], { options, debugMessages });
 		}
 		if (blocked) {
 			try {
-				await Fetch.failRequest({ requestId, errorReason: ABORTED_ERROR_REASON });
+				await Fetch.failRequest({ requestId, errorReason: ABORTED_ERROR_REASON }, sessionId);
 				return;
 			} catch {
 				// ignored
 			}
 		}
 		try {
-			await Fetch.continueRequest({ requestId });
+			await Fetch.continueRequest({ requestId }, sessionId);
 		} catch {
 			// ignored
 		}
@@ -381,13 +391,13 @@ function captureHttpInfo(params, urlState, { options, debugMessages, httpInfo })
 	}
 }
-async function setupHttpHeaders({ Network }, { options, debugMessages }) {
+async function setupHttpHeaders({ Network }, { options, debugMessages }, sessionId) {
 	logData(["Setting HTTP headers", JSON.stringify(options.httpHeaders)], { options, debugMessages });
-	await Network.enable();
-	await Network.setExtraHTTPHeaders({ headers: options.httpHeaders });
+	await Network.enable({}, sessionId);
+	await Network.setExtraHTTPHeaders({ headers: options.httpHeaders }, sessionId);
 }
-async function setupMediaFeatures({ Emulation }, { options, debugMessages }) {
+async function setupMediaFeatures({ Emulation }, { options, debugMessages }, sessionId) {
 	const features = [];
 	for (const mediaFeature of options.emulateMediaFeatures) {
 		logData(["Emulating media feature", mediaFeature.name, mediaFeature.value], { options, debugMessages });
@@ -395,7 +405,7 @@ async function setupMediaFeatures({ Emulation }, { options, debugMessages }) {
 			features.push({ name: mediaFeature.name, value: value.trim() });
 		}
 	}
-	await Emulation.setEmulatedMedia({ features });
+	await Emulation.setEmulatedMedia({ features }, sessionId);
 }
 async function setupCookies({ Network }, { options, debugMessages }) {
@@ -403,16 +413,77 @@ async function setupCookies({ Network }, { options, debugMessages }) {
 	await Network.setCookies({ cookies: options.browserCookies });
 }
-async function setupScriptInjection({ Page }, { options }) {
+async function setupScriptInjection(cdp, { options, debugMessages }) {
+	const { Page } = cdp;
+	const scriptSource = await getScriptSource(options);
 	await Page.addScriptToEvaluateOnNewDocument({
 		source: getHookScriptSource(),
 		runImmediately: true
 	});
 	await Page.addScriptToEvaluateOnNewDocument({
-		source: await getScriptSource(options),
+		source: scriptSource,
 		runImmediately: true,
 		worldName: SINGLE_FILE_WORLD_NAME
 	});
+	await setupFrameScriptInjection(cdp, scriptSource, { options, debugMessages });
+}
+// out-of-process frames are separate targets, so the scripts registered above
+// do not reach them; they are attached paused, injected and resumed instead
+async function setupFrameScriptInjection(cdp, scriptSource, { options, debugMessages }) {
+	const { Page, Runtime, Target } = cdp;
+	const ATTACHED_TO_TARGET_EVENT_TYPE = "attachedToTarget";
+	const IFRAME_TARGET_TYPE = "iframe";
+	const autoAttachOptions = { autoAttach: true, waitForDebuggerOnStart: true, flatten: true };
+	Target.addEventListener(ATTACHED_TO_TARGET_EVENT_TYPE, ignoringErrors(async ({ params }) => {
+		const { sessionId, targetInfo, waitingForDebugger } = params;
+		try {
+			if (targetInfo.type === IFRAME_TARGET_TYPE) {
+				logData(["Injecting scripts into frame", targetInfo.url], { options, debugMessages });
+				// the scripts registered below only run when the Page domain is
+				// enabled on the session
+				await Page.enable({}, sessionId);
+				await Target.setAutoAttach(autoAttachOptions, sessionId);
+				await Page.addScriptToEvaluateOnNewDocument({
+					source: getHookScriptSource(),
+					runImmediately: true
+				}, sessionId);
+				await Page.addScriptToEvaluateOnNewDocument({
+					source: scriptSource,
+					runImmediately: true,
+					worldName: SINGLE_FILE_WORLD_NAME
+				}, sessionId);
+				await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME }, sessionId);
+				await setupFrameSession(cdp, sessionId, { options, debugMessages });
+			}
+		} finally {
+			// the target is always resumed, even when a setup command fails,
+			// otherwise it would stay paused until the load timeout expires
+			if (waitingForDebugger) {
+				await Runtime.runIfWaitingForDebugger({}, sessionId);
+			}
+		}
+	}, { options, debugMessages }));
+	await Target.setAutoAttach(autoAttachOptions);
+}
+async function setupFrameSession({ Browser, Emulation, Fetch, Network, Security }, sessionId, { options, debugMessages }) {
+	const { handleAuthRequests, patterns } = getInterceptionOptions(options);
+	if (handleAuthRequests || (options.blockedURLPatterns && options.blockedURLPatterns.length)) {
+		// the requestPaused and authRequired listeners are registered once on
+		// the connection and receive the events of every session
+		await Fetch.enable({ handleAuthRequests, patterns }, sessionId);
+	}
+	await setupSecurity({ Security }, { options, debugMessages }, sessionId);
+	if (options.httpHeaders) {
+		await setupHttpHeaders({ Network }, { options, debugMessages }, sessionId);
+	}
+	if (options.emulateMediaFeatures) {
+		await setupMediaFeatures({ Emulation }, { options, debugMessages }, sessionId);
+	}
+	if (needsUserAgentOverride(options)) {
+		await setupUserAgent({ Browser, Emulation }, { options, debugMessages }, sessionId);
+	}
 }
 async function getContextId({ Debugger, Page, Runtime }, { options, debugMessages }) {
@@ -440,6 +511,9 @@ async function loadPage({ Page, Runtime }, { options, debugMessages }) {
 	const LOAD_TIMEOUT_ERROR_MESSAGE = "Load timeout";
 	await Runtime.enable();
 	await Page.enable();
+	if (options.browserBypassCSP) {
+		await Page.setBypassCSP({ enabled: true });
+	}
 	await Page.setLifecycleEventsEnabled({ enabled: true });
 	// the ID of the top frame is stable, and it is read before the navigation is
 	// triggered so that no event is missed while the browser answers
@@ -535,7 +609,8 @@ async function getSingleFileContext({ Page, Runtime }, topFrameId, { options, de
 	// injected script ran in instead of creating another one
 	const { executionContextId } = await Page.createIsolatedWorld({
 		frameId: topFrameId,
-		worldName: SINGLE_FILE_WORLD_NAME
+		worldName: SINGLE_FILE_WORLD_NAME,
+		contentSecurityPolicy: ""
 	});
 	// an empty world is returned when the script could not be injected, so the
 	// context is checked before it is used to capture the page
@@ -549,7 +624,7 @@ async function getSingleFileContext({ Page, Runtime }, topFrameId, { options, de
 	return executionContextId;
 }
-function setupPageDataCapture({ Runtime }, _contextId, { options, debugMessages }) {
+function setupPageDataCapture({ Runtime }, { options, debugMessages }) {
 	return new Promise((resolve, reject) => {
 		let pageDataResponse = "";
 		Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ({ params }) => {
@@ -574,33 +649,33 @@ function setupPageDataCapture({ Runtime }, _contextId, { options, debugMessages
 	});
 }
-async function setupBindings({ Page, Runtime }, contextId, { options, debugMessages, fetchAbortController }) {
-	await Runtime.addBinding({ name: SET_PAGE_DATA_FUNCTION_NAME, executionContextId: contextId });
+async function setupBindings({ Page, Runtime }, { options, debugMessages, blockedURLPatterns, fetchAbortController }) {
+	await Runtime.addBinding({ name: SET_PAGE_DATA_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
 	if (options.embedScreenshot && options.compressContent) {
-		await setupScreenshotCapture({ Page, Runtime }, contextId, { options, debugMessages });
+		await setupScreenshotCapture({ Page, Runtime }, { options, debugMessages });
 	}
 	if (options.embedPdf && options.compressContent) {
-		await setupPdfCapture({ Page, Runtime }, contextId, { options, debugMessages });
+		await setupPdfCapture({ Page, Runtime }, { options, debugMessages });
 	}
-	await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextId: contextId });
-	Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
+	await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
+	Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params, sessionId }) => {
 		if (params.name === FETCH_FUNCTION_NAME) {
-			await handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages, fetchAbortController });
+			await handleFetchRequest({ Runtime }, params, { options, debugMessages, blockedURLPatterns, fetchAbortController }, sessionId);
 		}
 	}, { options, debugMessages }));
 }
-async function setupScreenshotCapture({ Page, Runtime }, contextId, { options, debugMessages }) {
-	await Runtime.addBinding({ name: CAPTURE_SCREENSHOT_FUNCTION_NAME, executionContextId: contextId });
+async function setupScreenshotCapture({ Page, Runtime }, { options, debugMessages }) {
+	await Runtime.addBinding({ name: CAPTURE_SCREENSHOT_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
 	Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
 		if (params.name === CAPTURE_SCREENSHOT_FUNCTION_NAME) {
 			logData(["Capturing screenshot"], { options, debugMessages });
 			try {
 				const screenshotOptions = parseScreenshotOptions(options.embedScreenshotOptions);
 				const { data } = await Page.captureScreenshot(screenshotOptions);
-				await callBrowserFunction({ Runtime }, contextId, SET_SCREENSHOT_FUNCTION_NAME, [data]);
+				await callBrowserFunction({ Runtime }, params.executionContextId, SET_SCREENSHOT_FUNCTION_NAME, [data]);
 			} catch {
-				await callBrowserFunction({ Runtime }, contextId, SET_SCREENSHOT_FUNCTION_NAME, [""]);
+				await callBrowserFunction({ Runtime }, params.executionContextId, SET_SCREENSHOT_FUNCTION_NAME, [""]);
 			}
 		}
 	}, { options, debugMessages }));
@@ -620,17 +695,17 @@ async function setupScreenshotCapture({ Page, Runtime }, contextId, { options, d
 	}
 }
-async function setupPdfCapture({ Page, Runtime }, contextId, { options, debugMessages }) {
-	await Runtime.addBinding({ name: PRINT_TO_PDF_FUNCTION_NAME, executionContextId: contextId });
+async function setupPdfCapture({ Page, Runtime }, { options, debugMessages }) {
+	await Runtime.addBinding({ name: PRINT_TO_PDF_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
 	Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
 		if (params.name === PRINT_TO_PDF_FUNCTION_NAME) {
 			logData(["Printing to PDF", options.embedPdfOptions || ""], { options, debugMessages });
 			const pdfOptions = parsePdfOptions(options.embedPdfOptions);
 			try {
 				const { data } = await Page.printToPDF(pdfOptions);
-				await callBrowserFunction({ Runtime }, contextId, SET_PDF_FUNCTION_NAME, [data]);
+				await callBrowserFunction({ Runtime }, params.executionContextId, SET_PDF_FUNCTION_NAME, [data]);
 			} catch {
-				await callBrowserFunction({ Runtime }, contextId, SET_PDF_FUNCTION_NAME, [""]);
+				await callBrowserFunction({ Runtime }, params.executionContextId, SET_PDF_FUNCTION_NAME, [""]);
 			}
 		}
 	}, { options, debugMessages }));
@@ -648,12 +723,23 @@ async function setupPdfCapture({ Page, Runtime }, contextId, { options, debugMes
 	}
 }
-async function handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages, fetchAbortController }) {
-	const { payload } = params;
+async function handleFetchRequest({ Runtime }, params, { options, debugMessages, blockedURLPatterns, fetchAbortController }, sessionId) {
+	const BLOCKED_URL_ERROR_MESSAGE = "Blocked URL";
+	const { executionContextId: contextId, payload } = params;
 	const { requestId, url, options: fetchOptions } = JSON.parse(payload);
 	logData(["Fetching URL", url], { options, debugMessages });
 	try {
-		const response = await fetch(url, Object.assign({}, fetchOptions, { signal: fetchAbortController.signal }));
+		// this fetch does not go through the browser, so the URL blocking and
+		// the extra HTTP headers must be applied here too
+		if (blockedURLPatterns.some(pattern => pattern.test(url))) {
+			logData(["Blocking request", url], { options, debugMessages });
+			throw new Error(BLOCKED_URL_ERROR_MESSAGE);
+		}
+		const headers = Object.assign({}, fetchOptions.headers, options.httpHeaders);
+		if (options.userAgent) {
+			headers["user-agent"] = options.userAgent;
+		}
+		const response = await fetch(url, Object.assign({}, fetchOptions, { headers, signal: fetchAbortController.signal }));
 		const arrayBuffer = await response.arrayBuffer();
 		const base64Data = arrayBufferToBase64(arrayBuffer);
 		const result = {
@@ -661,22 +747,22 @@ async function handleFetchRequest({ Runtime }, params, contextId, { options, deb
 			headers: Object.fromEntries(response.headers.entries()),
 			data: base64Data
 		};
-		await callBrowserFunction({ Runtime }, contextId, RESOLVE_FETCH_FUNCTION_NAME, [requestId, result]);
+		await callBrowserFunction({ Runtime }, contextId, RESOLVE_FETCH_FUNCTION_NAME, [requestId, result], sessionId);
 	} catch (error) {
 		const errorResult = {
 			error: error.message,
 			code: error.code
 		};
-		await callBrowserFunction({ Runtime }, contextId, REJECT_FETCH_FUNCTION_NAME, [requestId, errorResult]);
+		await callBrowserFunction({ Runtime }, contextId, REJECT_FETCH_FUNCTION_NAME, [requestId, errorResult], sessionId);
 	}
 }
-async function callBrowserFunction({ Runtime }, contextId, functionName, args) {
+async function callBrowserFunction({ Runtime }, contextId, functionName, args, sessionId) {
 	const serializedArgs = args.map(arg => JSON.stringify(arg)).join(", ");
 	await Runtime.evaluate({
 		expression: `globalThis.${functionName}(${serializedArgs})`,
 		contextId
-	});
+	}, sessionId);
 }
 async function capturePageData({ Runtime }, contextId, { options, debugMessages }) {