single-file-cli 2.6.0 → 2.6.2

package/.github/workflows/ci.yml

@@ -28,3 +28,27 @@ jobs:
       -
         name: Run tests
         run: npm test
+
+  # lib/ holds generated code that is committed, so nothing at runtime notices when it falls behind
+  # the core release it was built from — a stale bundle just silently runs old code, and a dev build
+  # (build-dev.sh, unminified, built from a local core checkout) silently replaces the released one.
+  # build.sh is reproducible: it pins both the core package and esbuild, so rebuilding here and
+  # diffing catches either case before it ships.
+  build-output:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      -
+        name: Checkout sources
+        uses: actions/checkout@v7
+      -
+        name: Set up Deno
+        uses: denoland/setup-deno@v2
+        with:
+          deno-version: v2.x
+      -
+        name: Rebuild the released artifacts
+        run: bash build.sh
+      -
+        name: Check lib/ matches the pinned core release
+        run: git diff --exit-code --stat -- lib/

package/build-dev.sh

@@ -16,7 +16,7 @@ await build({
   ],
   bundle: true,
   globalName: 'singlefile',
-  outdir: 'lib/',
+  outdir: 'lib-dev/',
   platform: 'browser',
   sourcemap: false,
   minify: false,
@@ -30,7 +30,7 @@ await build({
   ],
   bundle: true,
   globalName: 'singlefileBootstrap',
-  outdir: 'lib/',
+  outdir: 'lib-dev/',
   platform: 'browser',
   sourcemap: false,
   minify: false,
@@ -43,7 +43,7 @@ await build({
     '../single-file-core/single-file-hooks-frames.js'
   ],
   bundle: true,
-  outdir: 'lib/',
+  outdir: 'lib-dev/',
   platform: 'browser',
   sourcemap: false,
   minify: false,
@@ -57,7 +57,7 @@ await build({
   ],
   bundle: true,
   globalName: 'zip',
-  outdir: 'lib/',
+  outdir: 'lib-dev/',
   platform: 'browser',
   sourcemap: false,
   minify: false,
@@ -70,7 +70,7 @@ await build({
     '../single-file-core/single-file-archive.js'
   ],
   bundle: true,
-  outfile: 'lib/single-file-archive.js',
+  outfile: 'lib-dev/single-file-archive.js',
   platform: 'neutral',
   sourcemap: false,
   minify: false,
@@ -79,23 +79,44 @@ await build({
 });
 
 const SCRIPTS = [
-	'lib/single-file.js',
-	'lib/single-file-bootstrap.js',
-	'lib/zip.min.js'
+	'lib-dev/single-file.js',
+	'lib-dev/single-file-bootstrap.js',
+	'lib-dev/zip.min.js'
 ];
 
 let script = '';
 const scripts = SCRIPTS.map(script => Deno.readTextFile(script));
 const sources = await Promise.all(scripts);
 script += 'const script = ' + JSON.stringify(sources.join(';')) + ';';
-const hookScript = await Deno.readTextFile('lib/single-file-hooks-frames.js');
+const hookScript = await Deno.readTextFile('lib-dev/single-file-hooks-frames.js');
 script += 'const hookScript = ' + JSON.stringify(hookScript) + ';';
-const zipScript = await Deno.readTextFile('lib/zip.min.js');
+const zipScript = await Deno.readTextFile('lib-dev/zip.min.js');
 script += 'const zipScript = ' + JSON.stringify(zipScript) + ';';
 script += 'export { script, zipScript, hookScript };';
-await Deno.writeTextFile('lib/single-file-bundle.js', script)
+await Deno.writeTextFile('lib-dev/single-file-bundle.js', script)
 await Promise.all(SCRIPTS.map(script => Deno.remove(script)));
-await Deno.remove('lib/single-file-hooks-frames.js');
+await Deno.remove('lib-dev/single-file-hooks-frames.js');
 const version = JSON.parse(await Deno.readTextFile('./deno.json')).version;
-await Deno.writeTextFile('lib/version.js', 'export const version = ' + JSON.stringify(version) + ';');
-" |  deno run --allow-read --allow-write --allow-net --allow-run --allow-env --lock="$build_dir/build.lock" -
+await Deno.writeTextFile('lib-dev/version.js', 'export const version = ' + JSON.stringify(version) + ';');
+" |  deno run --allow-read --allow-write --allow-net --allow-run --allow-env --lock="$build_dir/build.lock" -
+
+# Stage a runnable tree in .dev/, with the freshly built artifacts overlaid on the hand-written
+# files from lib/. Generated code never lands in the tracked lib/ directory, so a dev build cannot
+# leave an unminified bundle where the release one belongs, and using the dev build is something you
+# opt into by invoking .dev/single-file rather than something you get by forgetting to rebuild.
+rm -rf .dev
+mkdir -p .dev
+cp ./single-file ./single-file-node.js ./single-file-launcher.js ./single-file-cli-api.js ./options.js ./deno.json ./package.json .dev/
+cp -R ./resources .dev/resources
+cp -R ./lib .dev/lib
+cp lib-dev/* .dev/lib/
+
+core_revision=$(git -C ../single-file-core rev-parse --short HEAD 2>/dev/null || echo "unknown")
+if [ -n "$(git -C ../single-file-core status --porcelain 2>/dev/null | head -1)" ]; then
+	core_revision="$core_revision+uncommitted"
+fi
+
+echo
+echo "Dev build ready: single-file-core $core_revision"
+echo "  run it with:  node .dev/single-file <url> <output>"
+echo "  tracked lib/ is untouched; .dev/ and lib-dev/ are git-ignored"

package/build.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-CORE_PACKAGE="npm:single-file-core@1.5.113"
+CORE_PACKAGE="npm:single-file-core@1.5.116"
 ESBUILD_PACKAGE="npm:esbuild@0.27.7"
 
 project_dir=$(pwd)

package/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.6.0",
+  "version": "2.6.2",
   "description": "SingleFile CLI",
   "exports": {
     ".": "./single-file-cli-api.js"

package/eslint.config.mjs

@@ -1,7 +1,7 @@
 import js from "@eslint/js";
 
 export default [
-	{ ignores: ["lib/single-file-archive.js"] },
+	{ ignores: ["lib/single-file-archive.js", "lib-dev/**", ".dev/**"] },
 	js.configs.recommended,
 	{
 		languageOptions: {

package/lib/cdp-client.js

@@ -31,7 +31,7 @@ import {
 } from "./browser.js";
 import {
 	CDP,
-	options
+	options as cdpOptions
 } from "simple-cdp";
 import {
 	FETCH_FUNCTION_NAME,
@@ -66,6 +66,7 @@ const CORS_SAFELISTED_HEADER_NAMES = ["accept", "accept-language", "content-lang
 const CORS_UNSAFE_HEADER_VALUE = /[^\t\x20-\x7e]|[():<>?@[\\\]{}]/;
 const CORS_SAFELISTED_HEADER_VALUE_MAX_LENGTH = 128;
 const SERVICE_WORKER_TARGET_TYPE = "service_worker";
+const USER_AGENT_HEADER_NAME = "user-agent";
 
 let browserOptions, relaunchBrowserPromise;
 
@@ -77,10 +78,10 @@ export {
 
 async function initialize(singleFileOptions) {
 	if (singleFileOptions.browserServer) {
-		options.apiUrl = singleFileOptions.browserServer;
+		cdpOptions.apiUrl = singleFileOptions.browserServer;
 	} else {
 		browserOptions = getBrowserOptions(singleFileOptions);
-		options.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
+		cdpOptions.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
 	}
 }
 
@@ -90,7 +91,7 @@ function relaunchBrowser() {
 		relaunchBrowserPromise = (async () => {
 			await closeBrowser();
 			browserOptions.singleProcess = false;
-			options.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
+			cdpOptions.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
 		})();
 	}
 	return relaunchBrowserPromise;
@@ -101,7 +102,7 @@ async function getPageData(options) {
 	// compiled here so that an invalid pattern is reported before the page is
 	// loaded, instead of throwing for every request that is intercepted
 	const blockedURLPatterns = (options.blockedURLPatterns || []).map(pattern => new RegExp(pattern));
-	const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {}, blockedURLPatterns, fetchAbortController: new AbortController() };
+	const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {}, browserInfo: {}, blockedURLPatterns, fetchAbortController: new AbortController() };
 	let targetInfo, cdp;
 	try {
 		logData(["Loading page", EMPTY_PAGE_URL], pageContext);
@@ -111,7 +112,8 @@ async function getPageData(options) {
 		await setupBrowserWindow(cdp, targetInfo.id, pageContext);
 		await setupSecurity(cdp, pageContext);
 		await setupDeviceEmulation(cdp, pageContext);
-		await setupNetworkInterception(cdp, pageContext);
+		await setupNetwork(cdp, pageContext);
+		await setupDownloadBehavior(cdp);
 		await setupScriptInjection(cdp, pageContext);
 		const contextId = await getContextId(cdp, pageContext);
 		const pageDataPromise = setupPageDataCapture(cdp, pageContext);
@@ -211,13 +213,18 @@ async function setupSecurity({ Security }, { options, debugMessages }, sessionId
 	}
 }
 
-async function setupDeviceEmulation({ Browser, Emulation, Runtime }, { options, debugMessages }) {
+async function setupDeviceEmulation({ Browser, Emulation, Runtime }, { options, debugMessages, browserInfo }) {
 	const needsDeviceMetrics = options.browserMobileEmulation || options.browserDeviceWidth ||
 		options.browserDeviceHeight || options.browserDeviceScaleFactor;
 	if (needsDeviceMetrics) {
 		await setupDeviceMetrics({ Emulation, Runtime }, { options, debugMessages });
 	}
-	await setupUserAgent({ Browser, Emulation }, { options, debugMessages });
+	if (options.emulateMediaFeatures) {
+		await setupMediaFeatures({ Emulation }, { options, debugMessages });
+	}
+	// kept so that the resources fetched outside the browser present the same
+	// identity as the browser itself
+	browserInfo.userAgent = await setupUserAgent({ Browser, Emulation }, { options, debugMessages });
 }
 
 function needsUserAgentOverride(options) {
@@ -261,14 +268,14 @@ async function setupUserAgent({ Browser, Emulation }, { options, debugMessages }
 		logData(["Emulating user agent", JSON.stringify(agentOptions)], { options, debugMessages });
 		await Emulation.setUserAgentOverride(agentOptions, sessionId);
 	}
+	return agentOptions.userAgent;
 }
 
 function removeHeadlessToken(userAgent) {
 	return userAgent.replace(HEADLESS_USER_AGENT_TOKEN, "");
 }
 
-async function setupNetworkInterception({ Browser, Emulation, Fetch, Network }, { options, debugMessages, httpInfo, blockedURLPatterns }) {
-	const DENY_BEHAVIOR = "deny";
+async function setupNetwork({ Fetch, Network }, { options, debugMessages, httpInfo, blockedURLPatterns }) {
 	const { handleAuthRequests, patterns } = getInterceptionOptions(options);
 	await Fetch.enable({ handleAuthRequests, patterns });
 	if (handleAuthRequests) {
@@ -278,12 +285,13 @@ async function setupNetworkInterception({ Browser, Emulation, Fetch, Network },
 	if (options.httpHeaders && !hasUnsafeHttpHeaders(options)) {
 		await setupHttpHeaders({ Network }, { options, debugMessages });
 	}
-	if (options.emulateMediaFeatures) {
-		await setupMediaFeatures({ Emulation }, { options, debugMessages });
-	}
 	if (options.browserCookies && options.browserCookies.length) {
 		await setupCookies({ Network }, { options, debugMessages });
 	}
+}
+
+async function setupDownloadBehavior({ Browser }) {
+	const DENY_BEHAVIOR = "deny";
 	await Browser.setDownloadBehavior({ behavior: DENY_BEHAVIOR });
 }
 
@@ -693,7 +701,7 @@ function setupPageDataCapture({ Runtime }, { options, debugMessages }) {
 	});
 }
 
-async function setupBindings({ Page, Runtime }, { options, debugMessages, blockedURLPatterns, fetchAbortController }) {
+async function setupBindings({ Page, Runtime }, { options, debugMessages, browserInfo, blockedURLPatterns, fetchAbortController }) {
 	await Runtime.addBinding({ name: SET_PAGE_DATA_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
 	if (options.embedScreenshot && options.compressContent) {
 		await setupScreenshotCapture({ Page, Runtime }, { options, debugMessages });
@@ -704,7 +712,7 @@ async function setupBindings({ Page, Runtime }, { options, debugMessages, blocke
 	await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
 	Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params, sessionId }) => {
 		if (params.name === FETCH_FUNCTION_NAME) {
-			await handleFetchRequest({ Runtime }, params, { options, debugMessages, blockedURLPatterns, fetchAbortController }, sessionId);
+			await handleFetchRequest({ Runtime }, params, { options, debugMessages, browserInfo, blockedURLPatterns, fetchAbortController }, sessionId);
 		}
 	}, { options, debugMessages }));
 }
@@ -767,7 +775,7 @@ async function setupPdfCapture({ Page, Runtime }, { options, debugMessages }) {
 	}
 }
 
-async function handleFetchRequest({ Runtime }, params, { options, debugMessages, blockedURLPatterns, fetchAbortController }, sessionId) {
+async function handleFetchRequest({ Runtime }, params, { options, debugMessages, browserInfo, blockedURLPatterns, fetchAbortController }, sessionId) {
 	const BLOCKED_URL_ERROR_MESSAGE = "Blocked URL";
 	const { executionContextId: contextId, payload } = params;
 	const { requestId, url, options: fetchOptions } = JSON.parse(payload);
@@ -779,9 +787,12 @@ async function handleFetchRequest({ Runtime }, params, { options, debugMessages,
 			logData(["Blocking request", url], { options, debugMessages });
 			throw new Error(BLOCKED_URL_ERROR_MESSAGE);
 		}
+		// this fetch runs outside the browser, so without the user agent below it
+		// would reach the server under the runtime's own identity while every
+		// request the browser made presented another one
 		const headers = Object.assign({}, fetchOptions.headers, options.httpHeaders);
-		if (options.userAgent) {
-			headers["user-agent"] = options.userAgent;
+		if (browserInfo.userAgent && !Object.keys(headers).some(name => name.toLowerCase() == USER_AGENT_HEADER_NAME)) {
+			headers[USER_AGENT_HEADER_NAME] = browserInfo.userAgent;
 		}
 		const response = await fetch(url, Object.assign({}, fetchOptions, { headers, signal: fetchAbortController.signal }));
 		const arrayBuffer = await response.arrayBuffer();