single-file-cli 2.5.0 → 2.6.1

package/.github/workflows/ci.yml

@@ -12,10 +12,10 @@ jobs:
     steps:
       -
         name: Checkout sources
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
       -
         name: Set up Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v7
         with:
           node-version: 24
           cache: npm

package/.github/workflows/publish.yml

@@ -22,7 +22,7 @@ jobs:
     steps:
       # only used to resolve the tag to a commit, so it runs no code from the repository and keeps no
       # git credentials behind
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7
         with:
           ref: ${{ github.event.release.tag_name }}
           persist-credentials: false
@@ -86,11 +86,11 @@ jobs:
     needs: check_tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7
         with:
           ref: ${{ github.event.release.tag_name }}
       # Setup .npmrc file to publish to npm
-      - uses: actions/setup-node@v5
+      - uses: actions/setup-node@v7
         with:
           node-version: '24'
           registry-url: 'https://registry.npmjs.org'
@@ -121,19 +121,19 @@ jobs:
           exit 1
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
       -
         name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build and push
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           push: true
           build-args: |

package/build.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-CORE_PACKAGE="npm:single-file-core@1.5.109"
+CORE_PACKAGE="npm:single-file-core@1.5.115"
 ESBUILD_PACKAGE="npm:esbuild@0.27.7"
 
 project_dir=$(pwd)

package/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.5.0",
+  "version": "2.6.1",
   "description": "SingleFile CLI",
   "exports": {
     ".": "./single-file-cli-api.js"

package/lib/browser.js

@@ -62,7 +62,7 @@ const PROFILE_INCOMPATIBLE_ARGS = [
 const { build, makeTempDir, readDir, copyFile, mkdir, Command, errors, remove } = Deno;
 const { join } = path;
 let child, profilePath, childExited, keepProfilePath, browserDebugPort;
-export { launchBrowser, createBrowserProfile, getBrowserOptions, copyProfile, closeBrowser, browserExited };
+export { launchBrowser, createBrowserProfile, getBrowserOptions, copyProfile, pruneProfile, closeBrowser, browserExited };
 
 function getBrowserOptions(options) {
 	return {
@@ -71,7 +71,6 @@ function getBrowserOptions(options) {
 		executablePath: options.browserExecutablePath,
 		debug: options.browserDebug,
 		singleProcess: options.browserSingleProcess,
-		disableWebSecurity: options.browserDisableWebSecurity,
 		width: options.browserWidth,
 		height: options.browserHeight,
 		userAgent: options.userAgent,
@@ -241,6 +240,7 @@ async function closeBrowser() {
 	if (closedProfilePath !== undefined) {
 		if (keepProfilePath) {
 			await removeProfileLockEntries(closedProfilePath).catch(() => { });
+			await pruneProfile(closedProfilePath).catch(() => { });
 		} else {
 			try {
 				await remove(closedProfilePath, { recursive: true });
@@ -298,6 +298,27 @@ async function copyProfileEntries(entries, sourcePath, destinationPath) {
 	}
 }
 
+// the caches the browser fills during the session are the bulk of the profile and
+// none of them is ever copied to the capture profile, so they are dropped rather
+// than kept and copied over at every run
+async function pruneProfile(profileDirectory) {
+	await pruneProfileEntries(await readDir(profileDirectory), profileDirectory);
+}
+
+async function pruneProfileEntries(entries, profileDirectory) {
+	for (const entry of entries) {
+		if (entry.isSymlink) {
+			continue;
+		}
+		const entryPath = join(profileDirectory, entry.name);
+		if (PROFILE_IGNORED_ENTRY_NAMES.includes(entry.name)) {
+			await remove(entryPath, { recursive: true }).catch(() => { });
+		} else if (entry.isDirectory) {
+			await pruneProfileEntries(await readDir(entryPath), entryPath);
+		}
+	}
+}
+
 async function removeProfileLockEntries(profileDirectory) {
 	const entries = await readDir(profileDirectory);
 	for (const entry of entries) {

package/lib/cdp-client.js

@@ -31,7 +31,7 @@ import {
 } from "./browser.js";
 import {
 	CDP,
-	options
+	options as cdpOptions
 } from "simple-cdp";
 import {
 	FETCH_FUNCTION_NAME,
@@ -62,6 +62,11 @@ const BINDING_CALLED_EVENT_TYPE = "bindingCalled";
 const LOCALHOST = "http://localhost:";
 const HEADLESS_USER_AGENT_TOKEN = "Headless";
 const BROWSER_EXITED_MAX_DELAY = 2000;
+const CORS_SAFELISTED_HEADER_NAMES = ["accept", "accept-language", "content-language"];
+const CORS_UNSAFE_HEADER_VALUE = /[^\t\x20-\x7e]|[():<>?@[\\\]{}]/;
+const CORS_SAFELISTED_HEADER_VALUE_MAX_LENGTH = 128;
+const SERVICE_WORKER_TARGET_TYPE = "service_worker";
+const USER_AGENT_HEADER_NAME = "user-agent";
 
 let browserOptions, relaunchBrowserPromise;
 
@@ -73,10 +78,10 @@ export {
 
 async function initialize(singleFileOptions) {
 	if (singleFileOptions.browserServer) {
-		options.apiUrl = singleFileOptions.browserServer;
+		cdpOptions.apiUrl = singleFileOptions.browserServer;
 	} else {
 		browserOptions = getBrowserOptions(singleFileOptions);
-		options.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
+		cdpOptions.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
 	}
 }
 
@@ -86,7 +91,7 @@ function relaunchBrowser() {
 		relaunchBrowserPromise = (async () => {
 			await closeBrowser();
 			browserOptions.singleProcess = false;
-			options.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
+			cdpOptions.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
 		})();
 	}
 	return relaunchBrowserPromise;
@@ -97,7 +102,7 @@ async function getPageData(options) {
 	// compiled here so that an invalid pattern is reported before the page is
 	// loaded, instead of throwing for every request that is intercepted
 	const blockedURLPatterns = (options.blockedURLPatterns || []).map(pattern => new RegExp(pattern));
-	const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {}, blockedURLPatterns, fetchAbortController: new AbortController() };
+	const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {}, browserInfo: {}, blockedURLPatterns, fetchAbortController: new AbortController() };
 	let targetInfo, cdp;
 	try {
 		logData(["Loading page", EMPTY_PAGE_URL], pageContext);
@@ -107,7 +112,8 @@ async function getPageData(options) {
 		await setupBrowserWindow(cdp, targetInfo.id, pageContext);
 		await setupSecurity(cdp, pageContext);
 		await setupDeviceEmulation(cdp, pageContext);
-		await setupNetworkInterception(cdp, pageContext);
+		await setupNetwork(cdp, pageContext);
+		await setupDownloadBehavior(cdp);
 		await setupScriptInjection(cdp, pageContext);
 		const contextId = await getContextId(cdp, pageContext);
 		const pageDataPromise = setupPageDataCapture(cdp, pageContext);
@@ -207,13 +213,18 @@ async function setupSecurity({ Security }, { options, debugMessages }, sessionId
 	}
 }
 
-async function setupDeviceEmulation({ Browser, Emulation, Runtime }, { options, debugMessages }) {
+async function setupDeviceEmulation({ Browser, Emulation, Runtime }, { options, debugMessages, browserInfo }) {
 	const needsDeviceMetrics = options.browserMobileEmulation || options.browserDeviceWidth ||
 		options.browserDeviceHeight || options.browserDeviceScaleFactor;
 	if (needsDeviceMetrics) {
 		await setupDeviceMetrics({ Emulation, Runtime }, { options, debugMessages });
 	}
-	await setupUserAgent({ Browser, Emulation }, { options, debugMessages });
+	if (options.emulateMediaFeatures) {
+		await setupMediaFeatures({ Emulation }, { options, debugMessages });
+	}
+	// kept so that the resources fetched outside the browser present the same
+	// identity as the browser itself
+	browserInfo.userAgent = await setupUserAgent({ Browser, Emulation }, { options, debugMessages });
 }
 
 function needsUserAgentOverride(options) {
@@ -257,29 +268,30 @@ async function setupUserAgent({ Browser, Emulation }, { options, debugMessages }
 		logData(["Emulating user agent", JSON.stringify(agentOptions)], { options, debugMessages });
 		await Emulation.setUserAgentOverride(agentOptions, sessionId);
 	}
+	return agentOptions.userAgent;
 }
 
 function removeHeadlessToken(userAgent) {
 	return userAgent.replace(HEADLESS_USER_AGENT_TOKEN, "");
 }
 
-async function setupNetworkInterception({ Browser, Emulation, Fetch, Network }, { options, debugMessages, httpInfo, blockedURLPatterns }) {
-	const DENY_BEHAVIOR = "deny";
+async function setupNetwork({ Fetch, Network }, { options, debugMessages, httpInfo, blockedURLPatterns }) {
 	const { handleAuthRequests, patterns } = getInterceptionOptions(options);
 	await Fetch.enable({ handleAuthRequests, patterns });
 	if (handleAuthRequests) {
 		setupProxyAuth({ Fetch }, { options, debugMessages });
 	}
 	setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo, blockedURLPatterns });
-	if (options.httpHeaders) {
+	if (options.httpHeaders && !hasUnsafeHttpHeaders(options)) {
 		await setupHttpHeaders({ Network }, { options, debugMessages });
 	}
-	if (options.emulateMediaFeatures) {
-		await setupMediaFeatures({ Emulation }, { options, debugMessages });
-	}
 	if (options.browserCookies && options.browserCookies.length) {
 		await setupCookies({ Network }, { options, debugMessages });
 	}
+}
+
+async function setupDownloadBehavior({ Browser }) {
+	const DENY_BEHAVIOR = "deny";
 	await Browser.setDownloadBehavior({ behavior: DENY_BEHAVIOR });
 }
 
@@ -287,12 +299,33 @@ function getInterceptionOptions(options) {
 	const REQUEST_STAGE = "Request";
 	const RESPONSE_STAGE = "Response";
 	const handleAuthRequests = Boolean(options.httpProxyUsername);
-	const patterns = handleAuthRequests ?
+	// a request paused on the response stage has already been sent and answered,
+	// so a blocked URL is only left uncontacted when the request stage is asked for
+	const patterns = handleAuthRequests || hasUnsafeHttpHeaders(options) || hasBlockedURLPatterns(options) ?
 		[{ requestStage: REQUEST_STAGE }, { requestStage: RESPONSE_STAGE }] :
 		[{ requestStage: RESPONSE_STAGE }];
 	return { handleAuthRequests, patterns };
 }
 
+function hasBlockedURLPatterns(options) {
+	return Boolean(options.blockedURLPatterns && options.blockedURLPatterns.length);
+}
+
+// headers set on the network layer are part of the CORS decision: one that is not
+// safelisted makes every cross-origin fetch of the page send a preflight, and the
+// servers that do not answer preflights lose the resource to the backend fetch.
+// Such headers are injected on the paused request instead, below that decision
+function hasUnsafeHttpHeaders(options) {
+	return Boolean(options.httpHeaders) &&
+		Object.entries(options.httpHeaders).some(([name, value]) => !isSafelistedHeader(name, value));
+}
+
+function isSafelistedHeader(name, value) {
+	return CORS_SAFELISTED_HEADER_NAMES.includes(name.toLowerCase()) &&
+		value.length <= CORS_SAFELISTED_HEADER_VALUE_MAX_LENGTH &&
+		!CORS_UNSAFE_HEADER_VALUE.test(value);
+}
+
 function setupProxyAuth({ Fetch }, { options, debugMessages }) {
 	const AUTH_REQUIRED_EVENT_TYPE = "authRequired";
 	const PROVIDE_CREDENTIALS_RESPONSE = "ProvideCredentials";
@@ -313,8 +346,12 @@ function setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo,
 	const REQUEST_PAUSED_EVENT_TYPE = "requestPaused";
 	const ABORTED_ERROR_REASON = "Aborted";
 	const urlState = { url: options.url, alternativeUrl: getAlternativeUrl(options.url) };
+	const injectedHeaders = hasUnsafeHttpHeaders(options) ? options.httpHeaders : undefined;
 	Fetch.addEventListener(REQUEST_PAUSED_EVENT_TYPE, ignoringErrors(async ({ params, sessionId }) => {
-		const { requestId, request } = params;
+		const { requestId, request, responseStatusCode, responseErrorReason } = params;
+		// the request pauses twice when both stages are intercepted, and the
+		// headers can only be replaced on the first pause
+		const requestStage = responseStatusCode === undefined && responseErrorReason === undefined;
 		// the request is always resumed below, otherwise the page waits for it
 		// until the load timeout expires
 		let blocked = false;
@@ -335,7 +372,8 @@ function setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo,
 			}
 		}
 		try {
-			await Fetch.continueRequest({ requestId }, sessionId);
+			const headers = requestStage && injectedHeaders ? getMergedHeaders(request.headers, injectedHeaders) : undefined;
+			await Fetch.continueRequest(headers ? { requestId, headers } : { requestId }, sessionId);
 		} catch {
 			// ignored
 		}
@@ -351,6 +389,15 @@ function setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo,
 	}
 }
 
+// continueRequest replaces the headers of the request instead of adding to them,
+// so the ones the browser prepared are kept and the extra ones merged into them
+function getMergedHeaders(requestHeaders, extraHeaders) {
+	const headers = new Map();
+	Object.entries(requestHeaders).forEach(([name, value]) => headers.set(name.toLowerCase(), { name, value }));
+	Object.entries(extraHeaders).forEach(([name, value]) => headers.set(name.toLowerCase(), { name, value }));
+	return Array.from(headers.values());
+}
+
 function captureHttpInfo(params, urlState, { options, debugMessages, httpInfo }) {
 	const REDIRECT_STATUS_CODES = [301, 302, 303, 307, 308];
 	const DOCUMENT_RESOURCE_TYPE = "Document";
@@ -424,8 +471,9 @@ async function setupScriptInjection(cdp, { options, debugMessages }) {
 	await setupFrameScriptInjection(cdp, scriptSource, { options, debugMessages });
 }
 
-// out-of-process frames are separate targets, so the scripts registered above
-// do not reach them; they are attached paused, injected and resumed instead
+// out-of-process frames and service workers are separate targets, so neither the
+// scripts registered above nor the network options reach them; they are attached
+// paused, set up and resumed instead
 async function setupFrameScriptInjection(cdp, scriptSource, { options, debugMessages }) {
 	const { Page, Runtime, Target } = cdp;
 	const ATTACHED_TO_TARGET_EVENT_TYPE = "attachedToTarget";
@@ -434,7 +482,13 @@ async function setupFrameScriptInjection(cdp, scriptSource, { options, debugMess
 	Target.addEventListener(ATTACHED_TO_TARGET_EVENT_TYPE, ignoringErrors(async ({ params }) => {
 		const { sessionId, targetInfo, waitingForDebugger } = params;
 		try {
-			if (targetInfo.type === IFRAME_TARGET_TYPE) {
+			if (targetInfo.type === SERVICE_WORKER_TARGET_TYPE) {
+				logData(["Applying network options to service worker", targetInfo.url], { options, debugMessages });
+				// the requests a worker makes on behalf of the page are answered
+				// for by its own target, and it is attached paused, so the options
+				// are in place before it can serve a single one
+				await setupNetworkSession(cdp, sessionId, { options, debugMessages });
+			} else if (targetInfo.type === IFRAME_TARGET_TYPE) {
 				logData(["Injecting scripts into frame", targetInfo.url], { options, debugMessages });
 				// the scripts registered below only run when the Page domain is
 				// enabled on the session
@@ -463,17 +517,21 @@ async function setupFrameScriptInjection(cdp, scriptSource, { options, debugMess
 	await Target.setAutoAttach(autoAttachOptions);
 }
 
-async function setupFrameSession({ Browser, Emulation, Fetch, Network, Security }, sessionId, { options, debugMessages }) {
+async function setupNetworkSession({ Fetch, Network }, sessionId, { options, debugMessages }) {
 	const { handleAuthRequests, patterns } = getInterceptionOptions(options);
-	if (handleAuthRequests || (options.blockedURLPatterns && options.blockedURLPatterns.length)) {
+	if (handleAuthRequests || hasUnsafeHttpHeaders(options) || hasBlockedURLPatterns(options)) {
 		// the requestPaused and authRequired listeners are registered once on
 		// the connection and receive the events of every session
 		await Fetch.enable({ handleAuthRequests, patterns }, sessionId);
 	}
-	await setupSecurity({ Security }, { options, debugMessages }, sessionId);
-	if (options.httpHeaders) {
+	if (options.httpHeaders && !hasUnsafeHttpHeaders(options)) {
 		await setupHttpHeaders({ Network }, { options, debugMessages }, sessionId);
 	}
+}
+
+async function setupFrameSession({ Browser, Emulation, Fetch, Network, Security }, sessionId, { options, debugMessages }) {
+	await setupNetworkSession({ Fetch, Network }, sessionId, { options, debugMessages });
+	await setupSecurity({ Security }, { options, debugMessages }, sessionId);
 	if (options.emulateMediaFeatures) {
 		await setupMediaFeatures({ Emulation }, { options, debugMessages }, sessionId);
 	}
@@ -643,7 +701,7 @@ function setupPageDataCapture({ Runtime }, { options, debugMessages }) {
 	});
 }
 
-async function setupBindings({ Page, Runtime }, { options, debugMessages, blockedURLPatterns, fetchAbortController }) {
+async function setupBindings({ Page, Runtime }, { options, debugMessages, browserInfo, blockedURLPatterns, fetchAbortController }) {
 	await Runtime.addBinding({ name: SET_PAGE_DATA_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
 	if (options.embedScreenshot && options.compressContent) {
 		await setupScreenshotCapture({ Page, Runtime }, { options, debugMessages });
@@ -654,7 +712,7 @@ async function setupBindings({ Page, Runtime }, { options, debugMessages, blocke
 	await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
 	Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params, sessionId }) => {
 		if (params.name === FETCH_FUNCTION_NAME) {
-			await handleFetchRequest({ Runtime }, params, { options, debugMessages, blockedURLPatterns, fetchAbortController }, sessionId);
+			await handleFetchRequest({ Runtime }, params, { options, debugMessages, browserInfo, blockedURLPatterns, fetchAbortController }, sessionId);
 		}
 	}, { options, debugMessages }));
 }
@@ -717,7 +775,7 @@ async function setupPdfCapture({ Page, Runtime }, { options, debugMessages }) {
 	}
 }
 
-async function handleFetchRequest({ Runtime }, params, { options, debugMessages, blockedURLPatterns, fetchAbortController }, sessionId) {
+async function handleFetchRequest({ Runtime }, params, { options, debugMessages, browserInfo, blockedURLPatterns, fetchAbortController }, sessionId) {
 	const BLOCKED_URL_ERROR_MESSAGE = "Blocked URL";
 	const { executionContextId: contextId, payload } = params;
 	const { requestId, url, options: fetchOptions } = JSON.parse(payload);
@@ -729,9 +787,12 @@ async function handleFetchRequest({ Runtime }, params, { options, debugMessages,
 			logData(["Blocking request", url], { options, debugMessages });
 			throw new Error(BLOCKED_URL_ERROR_MESSAGE);
 		}
+		// this fetch runs outside the browser, so without the user agent below it
+		// would reach the server under the runtime's own identity while every
+		// request the browser made presented another one
 		const headers = Object.assign({}, fetchOptions.headers, options.httpHeaders);
-		if (options.userAgent) {
-			headers["user-agent"] = options.userAgent;
+		if (browserInfo.userAgent && !Object.keys(headers).some(name => name.toLowerCase() == USER_AGENT_HEADER_NAME)) {
+			headers[USER_AGENT_HEADER_NAME] = browserInfo.userAgent;
 		}
 		const response = await fetch(url, Object.assign({}, fetchOptions, { headers, signal: fetchAbortController.signal }));
 		const arrayBuffer = await response.arrayBuffer();
@@ -808,7 +869,7 @@ async function disableCdpDomains({ Console, Fetch, Network, Page, Runtime }, { o
 	await Fetch.disable();
 	await Runtime.disable();
 	await Page.disable();
-	if (options.httpHeaders) {
+	if (options.httpHeaders && !hasUnsafeHttpHeaders(options)) {
 		await Network.disable();
 	}
 	if (options.consoleMessagesFile) {