single-file-cli 2.3.1 → 2.4.1

package/README.MD

@@ -124,6 +124,15 @@ Make sure Chrome or a Chromium-based browser is installed in the default folder.
   single-file https://www.wikipedia.org wikipedia.html
   ```
 
+  - Save a page needing a logged-in session. Create the profile first, log in
+    to the website in the browser window, and quit the browser to save it. Then
+    reuse the profile as many times as needed, it is copied and left unmodified.
+
+  ```sh
+  single-file --create-browser-profile ./profiles/example https://www.example.com/login
+  single-file https://www.example.com/account account.html --browser-profile=./profiles/example
+  ```
+
   - Save a list of URLs stored into `list-urls.txt` in the current folder
 
   ```sh

package/build.sh

@@ -1,23 +1,30 @@
 #!/usr/bin/env bash
 
-mv package.json package.json.tmp
-mv deno.json deno.json.tmp
-mv deno.lock deno.lock.tmp
-deno install --vendor --quiet --minimum-dependency-age=0 "npm:single-file-core@1.5.98"
-mv package.json.tmp package.json
-mv deno.json.tmp deno.json
-mv deno.lock.tmp deno.lock
+set -e
+
+CORE_PACKAGE="npm:single-file-core@1.5.107"
+ESBUILD_PACKAGE="npm:esbuild@0.27.7"
+
+project_dir=$(pwd)
+build_dir=$(mktemp -d)
+trap 'rm -rf "$build_dir"' EXIT
+
+cd "$build_dir"
+deno install --vendor --quiet --minimum-dependency-age=0 "$CORE_PACKAGE"
 
 echo "
-import { build } from 'npm:esbuild';
+import { build } from '$ESBUILD_PACKAGE';
+
+const core = '$build_dir/node_modules/single-file-core';
+const lib = '$project_dir/lib';
 
 await build({
   entryPoints: [
-    'node_modules/single-file-core/single-file.js'
+    core + '/single-file.js'
   ],
   bundle: true,
   globalName: 'singlefile',
-  outdir: 'lib/',
+  outdir: lib,
   platform: 'browser',
   sourcemap: false,
   minify: true,
@@ -27,11 +34,11 @@ await build({
 
 await build({
   entryPoints: [
-    'node_modules/single-file-core/single-file-bootstrap.js'
+    core + '/single-file-bootstrap.js'
   ],
   bundle: true,
   globalName: 'singlefileBootstrap',
-  outdir: 'lib/',
+  outdir: lib,
   platform: 'browser',
   sourcemap: false,
   minify: true,
@@ -41,10 +48,10 @@ await build({
 
 await build({
   entryPoints: [
-    'node_modules/single-file-core/single-file-hooks-frames.js'
+    core + '/single-file-hooks-frames.js'
   ],
   bundle: true,
-  outdir: 'lib/',
+  outdir: lib,
   platform: 'browser',
   sourcemap: false,
   minify: true,
@@ -54,11 +61,11 @@ await build({
 
 await build({
   entryPoints: [
-    'node_modules/single-file-core/vendor/zip/zip.min.js'
+    core + '/vendor/zip/zip.min.js'
   ],
   bundle: true,
   globalName: 'zip',
-  outdir: 'lib/',
+  outdir: lib,
   platform: 'browser',
   sourcemap: false,
   minify: true,
@@ -67,12 +74,11 @@ await build({
 });
 
 await build({
-  stdin: {
-    contents: \"export * from './processors/compression/compression.js'; export * from './vendor/zip/zip.js';\",
-    resolveDir: 'node_modules/single-file-core',
-  },
+  entryPoints: [
+    core + '/single-file-archive.js'
+  ],
   bundle: true,
-  outfile: 'lib/single-file-archive.js',
+  outfile: lib + '/single-file-archive.js',
   platform: 'neutral',
   sourcemap: false,
   minify: true,
@@ -81,25 +87,23 @@ await build({
 });
 
 const SCRIPTS = [
-	'lib/single-file.js',
-	'lib/single-file-bootstrap.js',
-	'lib/zip.min.js'
+	lib + '/single-file.js',
+	lib + '/single-file-bootstrap.js',
+	lib + '/zip.min.js'
 ];
 
 let script = '';
 const scripts = SCRIPTS.map(script => Deno.readTextFile(script));
 const sources = await Promise.all(scripts);
 script += 'const script = ' + JSON.stringify(sources.join(';')) + ';';
-const hookScript = await Deno.readTextFile('lib/single-file-hooks-frames.js');
+const hookScript = await Deno.readTextFile(lib + '/single-file-hooks-frames.js');
 script += 'const hookScript = ' + JSON.stringify(hookScript) + ';';
-const zipScript = await Deno.readTextFile('lib/zip.min.js');
+const zipScript = await Deno.readTextFile(lib + '/zip.min.js');
 script += 'const zipScript = ' + JSON.stringify(zipScript) + ';';
 script += 'export { script, zipScript, hookScript };';
-await Deno.writeTextFile('lib/single-file-bundle.js', script)
+await Deno.writeTextFile(lib + '/single-file-bundle.js', script)
 await Promise.all(SCRIPTS.map(script => Deno.remove(script)));
-await Deno.remove('lib/single-file-hooks-frames.js');
-const version = JSON.parse(await Deno.readTextFile('./deno.json')).version;
-await Deno.writeTextFile('lib/version.js', 'export const version = ' + JSON.stringify(version) + ';');
-" |  deno run --allow-read --allow-write --allow-net --allow-run --allow-env --lock=node_modules/deno.lock.tmp -
-
-rm -rf node_modules
+await Deno.remove(lib + '/single-file-hooks-frames.js');
+const version = JSON.parse(await Deno.readTextFile('$project_dir/deno.json')).version;
+await Deno.writeTextFile(lib + '/version.js', 'export const version = ' + JSON.stringify(version) + ';');
+" |  deno run --allow-read --allow-write --allow-net --allow-run --allow-env --lock="$build_dir/build.lock" -

package/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.3.1",
+  "version": "2.4.1",
   "description": "SingleFile CLI",
   "exports": {
     ".": "./single-file-cli-api.js"

package/dev-build.sh

@@ -1,14 +1,18 @@
 #!/usr/bin/env bash
 
-mkdir -p node_modules/single-file-core
-cp -R ../single-file-core/* node_modules/single-file-core
+set -e
+
+ESBUILD_PACKAGE="npm:esbuild@0.27.7"
+
+build_dir=$(mktemp -d)
+trap 'rm -rf "$build_dir"' EXIT
 
 echo "
-import { build } from 'npm:esbuild';
+import { build } from '$ESBUILD_PACKAGE';
 
 await build({
   entryPoints: [
-    'node_modules/single-file-core/single-file.js'
+    '../single-file-core/single-file.js'
   ],
   bundle: true,
   globalName: 'singlefile',
@@ -22,7 +26,7 @@ await build({
 
 await build({
   entryPoints: [
-    'node_modules/single-file-core/single-file-bootstrap.js'
+    '../single-file-core/single-file-bootstrap.js'
   ],
   bundle: true,
   globalName: 'singlefileBootstrap',
@@ -36,7 +40,7 @@ await build({
 
 await build({
   entryPoints: [
-    'node_modules/single-file-core/single-file-hooks-frames.js'
+    '../single-file-core/single-file-hooks-frames.js'
   ],
   bundle: true,
   outdir: 'lib/',
@@ -49,7 +53,7 @@ await build({
 
 await build({
   entryPoints: [
-    'node_modules/single-file-core/vendor/zip/zip.min.js'
+    '../single-file-core/vendor/zip/zip.min.js'
   ],
   bundle: true,
   globalName: 'zip',
@@ -62,10 +66,9 @@ await build({
 });
 
 await build({
-  stdin: {
-    contents: \"export * from './processors/compression/compression.js'; export * from './vendor/zip/zip.js';\",
-    resolveDir: 'node_modules/single-file-core',
-  },
+  entryPoints: [
+    '../single-file-core/single-file-archive.js'
+  ],
   bundle: true,
   outfile: 'lib/single-file-archive.js',
   platform: 'neutral',
@@ -95,6 +98,4 @@ await Promise.all(SCRIPTS.map(script => Deno.remove(script)));
 await Deno.remove('lib/single-file-hooks-frames.js');
 const version = JSON.parse(await Deno.readTextFile('./deno.json')).version;
 await Deno.writeTextFile('lib/version.js', 'export const version = ' + JSON.stringify(version) + ';');
-" |  deno run --allow-read --allow-write --allow-net --allow-run --allow-env --lock=node_modules/deno.lock.tmp -
-
-rm -rf node_modules
+" |  deno run --allow-read --allow-write --allow-net --allow-run --allow-env --lock="$build_dir/build.lock" -

package/lib/browser.js

@@ -21,27 +21,90 @@
  *   Source.
  */
 
-/* global fetch, setTimeout */
+/* global fetch, setTimeout, WebSocket */
 
 import { BROWSER_PATHS, BROWSER_ARGS } from "./constants.js";
-import { Deno } from "./deno-polyfill.js";
+import { Deno, path } from "./deno-polyfill.js";
 
 const NULL_STD_CONFIG = "null";
 const DEBUG_PORT_MIN = 9222;
 const DEBUG_PORT_RANGE = 256;
 const READY_TIMEOUT = 30000;
 const READY_RETRY_DELAY = 250;
+const CLOSE_TIMEOUT = 15000;
+const LOCALHOST = "http://localhost:";
+const PROFILE_LOCK_ENTRY_PREFIX = "Singleton";
+const PROFILE_IGNORED_ENTRY_NAMES = [
+	"AutofillAiModelCache",
+	"Cache",
+	"Code Cache",
+	"CacheStorage",
+	"Crashpad",
+	"DawnCache",
+	"DawnGraphiteCache",
+	"DawnWebGPUCache",
+	"GPUCache",
+	"GPUPersistentCache",
+	"GraphiteDawnCache",
+	"GrShaderCache",
+	"ShaderCache",
+	"component_crx_cache",
+	"extensions_crx_cache",
+	"optimization_guide_hint_cache_store",
+	"optimization_guide_model_store"
+];
+const PROFILE_INCOMPATIBLE_ARGS = [
+	"--no-startup-window",
+	"--bwsi",
+	"--deny-permission-prompts"
+];
 
-const { build, makeTempDir, Command, errors, remove } = Deno;
-let child, profilePath, childExited;
-export { launchBrowser, closeBrowser, browserExited };
+const { build, makeTempDir, readDir, copyFile, mkdir, Command, errors, remove } = Deno;
+const { join } = path;
+let child, profilePath, childExited, keepProfilePath, browserDebugPort;
+export { launchBrowser, createBrowserProfile, getBrowserOptions, copyProfile, closeBrowser, browserExited };
+
+function getBrowserOptions(options) {
+	return {
+		args: options.browserArgs,
+		headless: options.browserHeadless,
+		executablePath: options.browserExecutablePath,
+		debug: options.browserDebug,
+		singleProcess: options.browserSingleProcess,
+		disableWebSecurity: options.browserDisableWebSecurity,
+		width: options.browserWidth,
+		height: options.browserHeight,
+		userAgent: options.userAgent,
+		httpProxyServer: options.httpProxyServer,
+		profile: options.browserProfile
+	};
+}
+
+async function createBrowserProfile(options = {}) {
+	await mkdir(options.profile, { recursive: true });
+	await launchBrowser(Object.assign({}, options, { headless: false, persistProfile: true }));
+	if (child !== undefined) {
+		await child.status;
+	}
+	await closeBrowser();
+}
 
 async function launchBrowser(options = {}, indexPath = 0) {
 	const executablePath = options.executablePath || BROWSER_PATHS[build.os][indexPath];
 	let args = Array.from(BROWSER_ARGS);
 	const debugPort = await getDebugPort();
+	browserDebugPort = debugPort;
 	args.push("--remote-debugging-port=" + debugPort);
-	profilePath = await makeTempDir();
+	keepProfilePath = Boolean(options.persistProfile);
+	if (keepProfilePath) {
+		profilePath = options.profile;
+		args = args.filter(arg => !PROFILE_INCOMPATIBLE_ARGS.includes(arg));
+	} else {
+		profilePath = await makeTempDir();
+		if (options.profile) {
+			await copyProfile(options.profile, profilePath);
+		}
+	}
 	if (options.headless && !options.debug) {
 		args.push("--headless");
 	} else {
@@ -74,11 +137,16 @@ async function launchBrowser(options = {}, indexPath = 0) {
 		!args.includes("--headless")) {
 		args.push("--disable-site-isolation-trials");
 	}
+	if (options.startUrl) {
+		args.push(options.startUrl);
+	}
 	const command = new Command(executablePath, { args, stdout: NULL_STD_CONFIG, stderr: NULL_STD_CONFIG });
 	try {
 		child = await command.spawn();
 	} catch (error) {
-		await remove(profilePath, { recursive: true }).catch(() => { });
+		if (!keepProfilePath) {
+			await remove(profilePath, { recursive: true }).catch(() => { });
+		}
 		profilePath = undefined;
 		if (error instanceof errors.NotFound) {
 			if (!options.executablePath && indexPath + 1 < BROWSER_PATHS[build.os].length) {
@@ -155,21 +223,86 @@ function getRandomDebugPort() {
 }
 
 async function closeBrowser() {
-	if (child !== undefined) {
+	const closedChild = child;
+	const closedProfilePath = profilePath;
+	child = undefined;
+	profilePath = undefined;
+	if (closedChild !== undefined) {
+		if (keepProfilePath && !childExited) {
+			await closeBrowserGracefully(closedChild);
+		}
 		try {
-			child.kill();
+			closedChild.kill();
 		} catch {
 			// ignored
 		}
-		await child.status;
-		child = undefined;
+		await closedChild.status;
 	}
-	if (profilePath !== undefined) {
-		try {
-			await remove(profilePath, { recursive: true });
-			profilePath = undefined;
-		} catch {
-			console.log("Warning: failed to remove profile directory: " + profilePath); // eslint-disable-line no-console
+	if (closedProfilePath !== undefined) {
+		if (keepProfilePath) {
+			await removeProfileLockEntries(closedProfilePath).catch(() => { });
+		} else {
+			try {
+				await remove(closedProfilePath, { recursive: true });
+			} catch {
+				console.log("Warning: failed to remove profile directory: " + closedProfilePath); // eslint-disable-line no-console
+			}
+		}
+	}
+}
+
+async function closeBrowserGracefully(closedChild) {
+	try {
+		const response = await fetch(LOCALHOST + browserDebugPort + "/json/version");
+		const { webSocketDebuggerUrl } = await response.json();
+		const socket = new WebSocket(webSocketDebuggerUrl);
+		await new Promise((resolve, reject) => {
+			socket.addEventListener("open", resolve);
+			socket.addEventListener("error", reject);
+		});
+		socket.send(JSON.stringify({ id: 0, method: "Browser.close" }));
+		await Promise.race([
+			closedChild.status,
+			new Promise(resolve => setTimeout(resolve, CLOSE_TIMEOUT))
+		]);
+	} catch {
+		// ignored
+	}
+}
+
+async function copyProfile(sourcePath, destinationPath) {
+	let entries;
+	try {
+		entries = await readDir(sourcePath);
+	} catch (error) {
+		throw error instanceof errors.NotFound ?
+			new Error(`The browser profile directory was not found at ${JSON.stringify(sourcePath)}`) :
+			error;
+	}
+	await copyProfileEntries(entries, sourcePath, destinationPath);
+}
+
+async function copyProfileEntries(entries, sourcePath, destinationPath) {
+	for (const entry of entries) {
+		if (entry.isSymlink || entry.name.startsWith(PROFILE_LOCK_ENTRY_PREFIX) || PROFILE_IGNORED_ENTRY_NAMES.includes(entry.name)) {
+			continue;
+		}
+		const entrySourcePath = join(sourcePath, entry.name);
+		const entryDestinationPath = join(destinationPath, entry.name);
+		if (entry.isDirectory) {
+			await mkdir(entryDestinationPath, { recursive: true });
+			await copyProfileEntries(await readDir(entrySourcePath), entrySourcePath, entryDestinationPath);
+		} else {
+			await copyFile(entrySourcePath, entryDestinationPath);
+		}
+	}
+}
+
+async function removeProfileLockEntries(profileDirectory) {
+	const entries = await readDir(profileDirectory);
+	for (const entry of entries) {
+		if (entry.name.startsWith(PROFILE_LOCK_ENTRY_PREFIX)) {
+			await remove(join(profileDirectory, entry.name)).catch(() => { });
 		}
 	}
 }

package/lib/cdp-client.js

@@ -25,6 +25,7 @@
 
 import {
 	launchBrowser,
+	getBrowserOptions,
 	closeBrowser,
 	browserExited
 } from "./browser.js";
@@ -59,6 +60,7 @@ const SET_PDF_FUNCTION_NAME = "setPDF";
 const SET_PAGE_DATA_FUNCTION_NAME = "setPageData";
 const BINDING_CALLED_EVENT_TYPE = "bindingCalled";
 const LOCALHOST = "http://localhost:";
+const HEADLESS_USER_AGENT_TOKEN = "Headless";
 const BROWSER_EXITED_MAX_DELAY = 2000;
 
 let browserOptions, relaunchBrowserPromise;
@@ -73,17 +75,7 @@ async function initialize(singleFileOptions) {
 	if (singleFileOptions.browserServer) {
 		options.apiUrl = singleFileOptions.browserServer;
 	} else {
-		browserOptions = {};
-		browserOptions.args = singleFileOptions.browserArgs;
-		browserOptions.headless = singleFileOptions.browserHeadless;
-		browserOptions.executablePath = singleFileOptions.browserExecutablePath;
-		browserOptions.debug = singleFileOptions.browserDebug;
-		browserOptions.singleProcess = singleFileOptions.browserSingleProcess;
-		browserOptions.disableWebSecurity = singleFileOptions.browserDisableWebSecurity;
-		browserOptions.width = singleFileOptions.browserWidth;
-		browserOptions.height = singleFileOptions.browserHeight;
-		browserOptions.userAgent = singleFileOptions.userAgent;
-		browserOptions.httpProxyServer = singleFileOptions.httpProxyServer;
+		browserOptions = getBrowserOptions(singleFileOptions);
 		options.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
 	}
 }
@@ -221,9 +213,7 @@ async function setupDeviceEmulation({ Browser, Emulation, Runtime }, { options,
 	if (needsDeviceMetrics) {
 		await setupDeviceMetrics({ Emulation, Runtime }, { options, debugMessages });
 	}
-	if (needsUserAgentOverride(options)) {
-		await setupUserAgent({ Browser, Emulation }, { options, debugMessages });
-	}
+	await setupUserAgent({ Browser, Emulation }, { options, debugMessages });
 }
 
 function needsUserAgentOverride(options) {
@@ -253,9 +243,9 @@ async function setupDeviceMetrics({ Emulation, Runtime }, { options, debugMessag
 async function setupUserAgent({ Browser, Emulation }, { options, debugMessages }, sessionId) {
 	const ANDROID_PLATFORM = "Android";
 	const { userAgent, product } = await Browser.getVersion();
-	const defaultMobileUA = `Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) ${product} Mobile Safari/537.36`;
+	const defaultMobileUA = `Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) ${removeHeadlessToken(product)} Mobile Safari/537.36`;
 	const agentOptions = {
-		userAgent: options.userAgent || (options.browserMobileEmulation ? defaultMobileUA : userAgent)
+		userAgent: options.userAgent || (options.browserMobileEmulation ? defaultMobileUA : removeHeadlessToken(userAgent))
 	};
 	if (options.acceptLanguage) {
 		agentOptions.acceptLanguage = options.acceptLanguage;
@@ -263,8 +253,14 @@ async function setupUserAgent({ Browser, Emulation }, { options, debugMessages }
 	if (options.platform || options.browserMobileEmulation) {
 		agentOptions.platform = options.platform || ANDROID_PLATFORM;
 	}
-	logData(["Emulating user agent", JSON.stringify(agentOptions)], { options, debugMessages });
-	await Emulation.setUserAgentOverride(agentOptions, sessionId);
+	if (needsUserAgentOverride(options) || agentOptions.userAgent !== userAgent) {
+		logData(["Emulating user agent", JSON.stringify(agentOptions)], { options, debugMessages });
+		await Emulation.setUserAgentOverride(agentOptions, sessionId);
+	}
+}
+
+function removeHeadlessToken(userAgent) {
+	return userAgent.replace(HEADLESS_USER_AGENT_TOKEN, "");
 }
 
 async function setupNetworkInterception({ Browser, Emulation, Fetch, Network }, { options, debugMessages, httpInfo, blockedURLPatterns }) {
@@ -481,9 +477,7 @@ async function setupFrameSession({ Browser, Emulation, Fetch, Network, Security
 	if (options.emulateMediaFeatures) {
 		await setupMediaFeatures({ Emulation }, { options, debugMessages }, sessionId);
 	}
-	if (needsUserAgentOverride(options)) {
-		await setupUserAgent({ Browser, Emulation }, { options, debugMessages }, sessionId);
-	}
+	await setupUserAgent({ Browser, Emulation }, { options, debugMessages }, sessionId);
 }
 
 async function getContextId({ Debugger, Page, Runtime }, { options, debugMessages }) {

package/lib/constants.js

@@ -43,7 +43,6 @@ const BROWSER_ARGS = [
 	"--disable-renderer-backgrounding",
 	"--force-color-profile=srgb",
 	"--no-first-run",
-	"--enable-automation",
 	"--password-store=basic",
 	"--use-mock-keychain",
 	"--no-service-autorun",

package/lib/deno-polyfill.js

@@ -101,8 +101,10 @@ const DenoAPI = {
 	args,
 	readFile,
 	readTextFile,
+	readDir,
 	writeTextFile,
 	writeFile,
+	copyFile,
 	mkdir,
 	makeTempDir,
 	stat,
@@ -118,6 +120,7 @@ const DenoAPI = {
 
 const pathAPI = {
 	dirname,
+	join,
 	toFileUrl,
 	fromFileUrl,
 	SEPARATOR: DENO_RUNTIME_DETECTED ? path.SEPARATOR : path.sep
@@ -154,6 +157,41 @@ async function readTextFile(path) {
 	}
 }
 
+async function readDir(path) {
+	if (DENO_RUNTIME_DETECTED) {
+		const entries = [];
+		try {
+			for await (const entry of Deno.readDir(path)) {
+				entries.push({ name: entry.name, isDirectory: entry.isDirectory, isSymlink: entry.isSymlink });
+			}
+		} catch (error) {
+			throw mapNotFoundError(error);
+		}
+		return entries;
+	} else {
+		const fsPromise = await import(NODE_MODULES["fs"]);
+		try {
+			const entries = await fsPromise.readdir(path, { withFileTypes: true });
+			return entries.map(entry => ({ name: entry.name, isDirectory: entry.isDirectory(), isSymlink: entry.isSymbolicLink() }));
+		} catch (error) {
+			throw mapNotFoundError(error);
+		}
+	}
+}
+
+async function copyFile(sourcePath, destinationPath) {
+	if (DENO_RUNTIME_DETECTED) {
+		return Deno.copyFile(sourcePath, destinationPath);
+	} else {
+		const fsPromise = await import(NODE_MODULES["fs"]);
+		return fsPromise.copyFile(sourcePath, destinationPath);
+	}
+}
+
+function mapNotFoundError(error) {
+	return error.code == "ENOENT" || error.code == "ENOTDIR" ? new errors.NotFound(error.message) : error;
+}
+
 async function writeTextFile(path, data, options = {}) {
 	if (DENO_RUNTIME_DETECTED) {
 		return Deno.writeTextFile(path, data, options);
@@ -259,6 +297,10 @@ function dirname(filePath) {
 	return path.dirname(filePath);
 }
 
+function join(...filePaths) {
+	return path.join(...filePaths);
+}
+
 async function toFileUrl(filePath) {
 	if (DENO_RUNTIME_DETECTED) {
 		return path.toFileUrl(filePath);