single-file-cli 2.6.0 → 2.6.1

package/build.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-CORE_PACKAGE="npm:single-file-core@1.5.113"
+CORE_PACKAGE="npm:single-file-core@1.5.115"
 ESBUILD_PACKAGE="npm:esbuild@0.27.7"
 
 project_dir=$(pwd)

package/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.6.0",
+  "version": "2.6.1",
   "description": "SingleFile CLI",
   "exports": {
     ".": "./single-file-cli-api.js"

package/lib/cdp-client.js

@@ -31,7 +31,7 @@ import {
 } from "./browser.js";
 import {
 	CDP,
-	options
+	options as cdpOptions
 } from "simple-cdp";
 import {
 	FETCH_FUNCTION_NAME,
@@ -66,6 +66,7 @@ const CORS_SAFELISTED_HEADER_NAMES = ["accept", "accept-language", "content-lang
 const CORS_UNSAFE_HEADER_VALUE = /[^\t\x20-\x7e]|[():<>?@[\\\]{}]/;
 const CORS_SAFELISTED_HEADER_VALUE_MAX_LENGTH = 128;
 const SERVICE_WORKER_TARGET_TYPE = "service_worker";
+const USER_AGENT_HEADER_NAME = "user-agent";
 
 let browserOptions, relaunchBrowserPromise;
 
@@ -77,10 +78,10 @@ export {
 
 async function initialize(singleFileOptions) {
 	if (singleFileOptions.browserServer) {
-		options.apiUrl = singleFileOptions.browserServer;
+		cdpOptions.apiUrl = singleFileOptions.browserServer;
 	} else {
 		browserOptions = getBrowserOptions(singleFileOptions);
-		options.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
+		cdpOptions.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
 	}
 }
 
@@ -90,7 +91,7 @@ function relaunchBrowser() {
 		relaunchBrowserPromise = (async () => {
 			await closeBrowser();
 			browserOptions.singleProcess = false;
-			options.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
+			cdpOptions.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
 		})();
 	}
 	return relaunchBrowserPromise;
@@ -101,7 +102,7 @@ async function getPageData(options) {
 	// compiled here so that an invalid pattern is reported before the page is
 	// loaded, instead of throwing for every request that is intercepted
 	const blockedURLPatterns = (options.blockedURLPatterns || []).map(pattern => new RegExp(pattern));
-	const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {}, blockedURLPatterns, fetchAbortController: new AbortController() };
+	const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {}, browserInfo: {}, blockedURLPatterns, fetchAbortController: new AbortController() };
 	let targetInfo, cdp;
 	try {
 		logData(["Loading page", EMPTY_PAGE_URL], pageContext);
@@ -111,7 +112,8 @@ async function getPageData(options) {
 		await setupBrowserWindow(cdp, targetInfo.id, pageContext);
 		await setupSecurity(cdp, pageContext);
 		await setupDeviceEmulation(cdp, pageContext);
-		await setupNetworkInterception(cdp, pageContext);
+		await setupNetwork(cdp, pageContext);
+		await setupDownloadBehavior(cdp);
 		await setupScriptInjection(cdp, pageContext);
 		const contextId = await getContextId(cdp, pageContext);
 		const pageDataPromise = setupPageDataCapture(cdp, pageContext);
@@ -211,13 +213,18 @@ async function setupSecurity({ Security }, { options, debugMessages }, sessionId
 	}
 }
 
-async function setupDeviceEmulation({ Browser, Emulation, Runtime }, { options, debugMessages }) {
+async function setupDeviceEmulation({ Browser, Emulation, Runtime }, { options, debugMessages, browserInfo }) {
 	const needsDeviceMetrics = options.browserMobileEmulation || options.browserDeviceWidth ||
 		options.browserDeviceHeight || options.browserDeviceScaleFactor;
 	if (needsDeviceMetrics) {
 		await setupDeviceMetrics({ Emulation, Runtime }, { options, debugMessages });
 	}
-	await setupUserAgent({ Browser, Emulation }, { options, debugMessages });
+	if (options.emulateMediaFeatures) {
+		await setupMediaFeatures({ Emulation }, { options, debugMessages });
+	}
+	// kept so that the resources fetched outside the browser present the same
+	// identity as the browser itself
+	browserInfo.userAgent = await setupUserAgent({ Browser, Emulation }, { options, debugMessages });
 }
 
 function needsUserAgentOverride(options) {
@@ -261,14 +268,14 @@ async function setupUserAgent({ Browser, Emulation }, { options, debugMessages }
 		logData(["Emulating user agent", JSON.stringify(agentOptions)], { options, debugMessages });
 		await Emulation.setUserAgentOverride(agentOptions, sessionId);
 	}
+	return agentOptions.userAgent;
 }
 
 function removeHeadlessToken(userAgent) {
 	return userAgent.replace(HEADLESS_USER_AGENT_TOKEN, "");
 }
 
-async function setupNetworkInterception({ Browser, Emulation, Fetch, Network }, { options, debugMessages, httpInfo, blockedURLPatterns }) {
-	const DENY_BEHAVIOR = "deny";
+async function setupNetwork({ Fetch, Network }, { options, debugMessages, httpInfo, blockedURLPatterns }) {
 	const { handleAuthRequests, patterns } = getInterceptionOptions(options);
 	await Fetch.enable({ handleAuthRequests, patterns });
 	if (handleAuthRequests) {
@@ -278,12 +285,13 @@ async function setupNetworkInterception({ Browser, Emulation, Fetch, Network },
 	if (options.httpHeaders && !hasUnsafeHttpHeaders(options)) {
 		await setupHttpHeaders({ Network }, { options, debugMessages });
 	}
-	if (options.emulateMediaFeatures) {
-		await setupMediaFeatures({ Emulation }, { options, debugMessages });
-	}
 	if (options.browserCookies && options.browserCookies.length) {
 		await setupCookies({ Network }, { options, debugMessages });
 	}
+}
+
+async function setupDownloadBehavior({ Browser }) {
+	const DENY_BEHAVIOR = "deny";
 	await Browser.setDownloadBehavior({ behavior: DENY_BEHAVIOR });
 }
 
@@ -693,7 +701,7 @@ function setupPageDataCapture({ Runtime }, { options, debugMessages }) {
 	});
 }
 
-async function setupBindings({ Page, Runtime }, { options, debugMessages, blockedURLPatterns, fetchAbortController }) {
+async function setupBindings({ Page, Runtime }, { options, debugMessages, browserInfo, blockedURLPatterns, fetchAbortController }) {
 	await Runtime.addBinding({ name: SET_PAGE_DATA_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
 	if (options.embedScreenshot && options.compressContent) {
 		await setupScreenshotCapture({ Page, Runtime }, { options, debugMessages });
@@ -704,7 +712,7 @@ async function setupBindings({ Page, Runtime }, { options, debugMessages, blocke
 	await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
 	Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params, sessionId }) => {
 		if (params.name === FETCH_FUNCTION_NAME) {
-			await handleFetchRequest({ Runtime }, params, { options, debugMessages, blockedURLPatterns, fetchAbortController }, sessionId);
+			await handleFetchRequest({ Runtime }, params, { options, debugMessages, browserInfo, blockedURLPatterns, fetchAbortController }, sessionId);
 		}
 	}, { options, debugMessages }));
 }
@@ -767,7 +775,7 @@ async function setupPdfCapture({ Page, Runtime }, { options, debugMessages }) {
 	}
 }
 
-async function handleFetchRequest({ Runtime }, params, { options, debugMessages, blockedURLPatterns, fetchAbortController }, sessionId) {
+async function handleFetchRequest({ Runtime }, params, { options, debugMessages, browserInfo, blockedURLPatterns, fetchAbortController }, sessionId) {
 	const BLOCKED_URL_ERROR_MESSAGE = "Blocked URL";
 	const { executionContextId: contextId, payload } = params;
 	const { requestId, url, options: fetchOptions } = JSON.parse(payload);
@@ -779,9 +787,12 @@ async function handleFetchRequest({ Runtime }, params, { options, debugMessages,
 			logData(["Blocking request", url], { options, debugMessages });
 			throw new Error(BLOCKED_URL_ERROR_MESSAGE);
 		}
+		// this fetch runs outside the browser, so without the user agent below it
+		// would reach the server under the runtime's own identity while every
+		// request the browser made presented another one
 		const headers = Object.assign({}, fetchOptions.headers, options.httpHeaders);
-		if (options.userAgent) {
-			headers["user-agent"] = options.userAgent;
+		if (browserInfo.userAgent && !Object.keys(headers).some(name => name.toLowerCase() == USER_AGENT_HEADER_NAME)) {
+			headers[USER_AGENT_HEADER_NAME] = browserInfo.userAgent;
 		}
 		const response = await fetch(url, Object.assign({}, fetchOptions, { headers, signal: fetchAbortController.signal }));
 		const arrayBuffer = await response.arrayBuffer();