single-file-cli 2.0.68 → 2.0.69

package/README.MD

@@ -65,7 +65,7 @@ Make sure Chrome or a Chromium-based browser is installed in the default folder.
 
 - There are 3 ways to download the code of SingleFile, choose the one you prefer:
   
-  - Install with `npm` (`npm` is installed with Node.js) and run with `npx`
+  - Install with `npm` and run `single-file` via `npx` (`npm` and `npx` are installed with Node.js)
   
     ```sh
     npm install "single-file-cli"

package/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.0.68",
+  "version": "2.0.69",
   "description": "SingleFile CLI",
   "nodeModulesDir": true,
   "exports": {

package/lib/browser.js

@@ -92,7 +92,7 @@ async function launchBrowser(options = {}, indexPath = 0) {
 async function getDebugPort(port = getRandomDebugPort(), usedPorts = []) {
 	try {
 		await fetch("http://localhost:" + port + "/json/version");
-	} catch (error) {
+	} catch {
 		return port;
 	}
 	if (usedPorts.length < DEBUG_PORT_RANGE) {
@@ -121,7 +121,7 @@ async function closeBrowser() {
 		try {
 			await remove(profilePath, { recursive: true });
 			profilePath = undefined;
-		} catch (error) {
+		} catch {
 			console.log("Warning: failed to remove profile directory: " + profilePath); // eslint-disable-line no-console
 		}
 	}

package/lib/cdp-client.js

@@ -159,7 +159,7 @@ async function getPageData(options) {
 			}
 			try {
 				await Fetch.continueRequest({ requestId });
-			} catch (error) {
+			} catch {
 				// ignored
 			}
 		});
@@ -177,6 +177,7 @@ async function getPageData(options) {
 		if (options.browserCookies && options.browserCookies.length) {
 			await Network.setCookies({ cookies: options.browserCookies });
 		}
+		await Browser.setDownloadBehavior({ behavior: "deny" });
 		await Page.addScriptToEvaluateOnNewDocument({
 			source: getHookScriptSource(),
 			runImmediately: true
@@ -200,14 +201,14 @@ async function getPageData(options) {
 						if (options.embedScreenshotOptions) {
 							try {
 								screenshotOptions = JSON.parse(options.embedScreenshotOptions);
-							} catch (error) {
+							} catch {
 								// ignored
 							}
 						}
 						screenshotOptions.format = "png";
 						const { data } = await Page.captureScreenshot(screenshotOptions);
 						await Runtime.evaluate({ expression: `globalThis.${SET_SCREENSHOT_FUNCTION_NAME}(${JSON.stringify(data)})`, contextId });
-					} catch (error) {
+					} catch {
 						await Runtime.evaluate({ expression: `globalThis.${SET_SCREENSHOT_FUNCTION_NAME}(${JSON.stringify("")})`, contextId });
 					}
 				}
@@ -221,14 +222,14 @@ async function getPageData(options) {
 					if (options.embedPdfOptions) {
 						try {
 							pdfOptions = JSON.parse(options.embedPdfOptions);
-						} catch (error) {
+						} catch {
 							// ignored
 						}
 					}
 					try {
 						const { data } = await Page.printToPDF(pdfOptions);
 						await Runtime.evaluate({ expression: `globalThis.${SET_PDF_FUNCTION_NAME}(${JSON.stringify(data)})`, contextId });
-					} catch (error) {
+					} catch {
 						await Runtime.evaluate({ expression: `globalThis.${SET_PDF_FUNCTION_NAME}(${JSON.stringify("")})`, contextId });
 					}
 				}
@@ -236,7 +237,7 @@ async function getPageData(options) {
 		}
 		const pageDataPromise = new Promise(resolve => {
 			let pageDataResponse = "";
-			Runtime.addEventListener("bindingCalled", async ({ params }) => {
+			Runtime.addEventListener("bindingCalled", ({ params }) => {
 				if (params.name === SET_PAGE_DATA_FUNCTION_NAME) {
 					const { payload } = params;
 					if (payload.length) {
@@ -276,13 +277,19 @@ async function getPageData(options) {
 			const browserWaitUntil = NETWORK_STATES[(NETWORK_STATES.indexOf(options.browserWaitUntil) + 1)];
 			if (browserWaitUntil) {
 				options.browserWaitUntil = browserWaitUntil;
+				await closeTarget();
 				return await getPageData(options);
 			}
 		}
 		throw error;
 	} finally {
+		await closeTarget();
+	}
+
+	async function closeTarget() {
 		if (targetInfo && !options.browserDebug) {
 			await CDP.closeTarget(targetInfo.id);
+			targetInfo = null;
 		}
 	}
 }
@@ -374,6 +381,7 @@ async function getTopFrameContextId({ Page, Runtime }, options) {
 	}
 
 	async function waitForPageReady({ Page }, options) {
+		let resolveCallbackTimeout;
 		await Page.setLifecycleEventsEnabled({ enabled: true });
 		try {
 			await new Promise((resolve, reject) => {
@@ -384,9 +392,12 @@ async function getTopFrameContextId({ Page, Runtime }, options) {
 
 				function onLifecycleEvent({ params }) {
 					const { frameId, name } = params;
-					if (frameId === topFrameId && name === options.browserWaitUntil) {
-						removeListeners();
-						resolve();
+					if (frameId === topFrameId && name === options.browserWaitUntil || (resolveCallbackTimeout && NETWORK_STATES.indexOf(name) < NETWORK_STATES.indexOf(options.browserWaitUntil))) {
+						clearTimeout(resolveCallbackTimeout);
+						setTimeout(() => {
+							removeListeners();
+							resolve();
+						}, options.browserWaitUntilDelay);
 					}
 				}
 
@@ -394,6 +405,7 @@ async function getTopFrameContextId({ Page, Runtime }, options) {
 					const { frame } = params;
 					if (!frame.parentId) {
 						if (frame.unreachableUrl) {
+							clearTimeout(resolveCallbackTimeout);
 							removeListeners();
 							reject(new Error("Unreachable URL: " + frame.unreachableUrl));
 						} else {
@@ -433,7 +445,7 @@ async function getTopFrameContextId({ Page, Runtime }, options) {
 				contextId
 			});
 			return result.value === true;
-		} catch (error) {
+		} catch {
 			// ignored
 		}
 		return false;

package/lib/deno-polyfill.js

@@ -21,6 +21,8 @@
  *   Source.
  */
 
+// deno-lint-ignore-file no-node-globals
+
 /* global globalThis, URL, Deno, process */
 
 import * as path from "path";
@@ -244,7 +246,7 @@ async function cwd() {
 	}
 }
 
-async function dirname(filePath) {
+function dirname(filePath) {
 	return path.dirname(filePath);
 }
 

package/lib/single-file-script.js

@@ -41,7 +41,7 @@ function initSingleFile() {
 				xhrRequest.onreadystatechange = () => {
 					if (xhrRequest.readyState == XMLHttpRequest.DONE) {
 						resolve({
-							arrayBuffer: async () => xhrRequest.response || new ArrayBuffer(),
+							arrayBuffer: () => xhrRequest.response || new ArrayBuffer(),
 							headers: { get: headerName => xhrRequest.getResponseHeader(headerName) },
 							status: xhrRequest.status
 						});

package/lib/version.js

@@ -1 +1 @@
-export const version = "2.0.68";
+export const version = "2.0.69";

package/options.js

@@ -50,6 +50,7 @@ const OPTIONS_INFO = {
 	"browser-load-max-time": { description: "Maximum delay of time to wait for page loading in ms", type: "number", defaultValue: 60000 },
 	"browser-wait-delay": { description: "Time to wait before capturing the page in ms", type: "number" },
 	"browser-wait-until": { description: "When to consider the page is loaded (InteractiveTime, networkIdle, networkAlmostIdle, load, domContentLoaded)", type: "string", defaultValue: "networkIdle" },
+	"browser-wait-until-delay": { description: "Delay of time in ms to wait before considering the page is loaded when the value of --browser-wait-until is reached", type: "number", defaultValue: 1000 },
 	"browser-wait-until-fallback": { description: "Retry with the next value of --browser-wait-until when a timeout error is thrown", type: "boolean", defaultValue: true },
 	"browser-debug": { description: "Enable debug mode", type: "boolean" },
 	"browser-script": { description: "Path of a script executed in the page (and all the frames) before it is loaded", type: "string[]" },
@@ -121,7 +122,7 @@ const OPTIONS_INFO = {
 	"remove-alternative-images": { description: "Remove images for alternative sizes of screen", type: "boolean", defaultValue: true },
 	"save-original-URLs": { description: "Save the original URLS in the embedded contents", type: "boolean" },
 	"save-raw-page": { description: "Save the original page without interpreting it into the browser", type: "boolean" },
-	"urls-file": { description: "Path to a text file containing a list of URLs (separated by a newline) to save", type: "string" },
+	"urls-file": { description: "Path to a text file containing a list of URLs (separated by a newline) to save. You can also pass the options after each URL (e.g. 'https://www.example.com --filename-template={page-title}.html')", type: "string" },
 	"user-agent": { description: "User-agent of the browser", type: "string" },
 	"accept-language": { description: "Accept language of the browser", type: "string" },
 	"platform": { description: "Platform of the browser (default value is \"Android\" when using --browser-mobile-emulation)", type: "string" },
@@ -149,10 +150,10 @@ const OPTIONS_INFO = {
 };
 
 const { args, exit } = Deno;
-const options = await getOptions();
-export default options;
+const options = getOptions();
+export { options, parseArgs };
 
-async function getOptions() {
+function getOptions() {
 	const { positionals, options } = parseArgs(Array.from(args));
 	const unknownOptions = [];
 	positionals.forEach(positional => {
@@ -185,43 +186,6 @@ async function getOptions() {
 		console.log(version); // eslint-disable-line no-console
 		exit(0);
 	}
-	Object.keys(OPTIONS_INFO).forEach(optionName => {
-		const optionInfo = getOptionInfo(optionName);
-		const optionKey = getOptionKey(optionName, optionInfo);
-		if (options[optionKey] === undefined && optionInfo.defaultValue !== undefined) {
-			options[optionKey] = OPTIONS_INFO[optionName].defaultValue;
-		}
-	});
-	options.acceptHeaders = {
-		font: options.acceptHeaderFont,
-		image: options.acceptHeaderImage,
-		stylesheet: options.acceptHeaderStylesheet,
-		script: options.acceptHeaderScript,
-		document: options.acceptHeaderDocument
-	};
-	if (options.browserArgs) {
-		const browserArguments = options.browserArguments || [];
-		browserArguments.push(...JSON.parse(options.browserArgs));
-		options.browserArgs = browserArguments;
-		delete options.browserArguments;
-	}
-	if (options.browserArguments) {
-		options.browserArgs = options.browserArguments;
-		delete options.browserArguments;
-	}
-	if (options.errorFile) {
-		options.errorsFile = options.errorFile;
-		delete options.errorFile;
-	}
-	if (options.errorTracesDisabled) {
-		options.errorsTracesDisabled = options.errorTracesDisabled;
-		delete options.errorTracesDisabled;
-	}
-	delete options.acceptHeaderFont;
-	delete options.acceptHeaderImage;
-	delete options.acceptHeaderStylesheet;
-	delete options.acceptHeaderScript;
-	delete options.acceptHeaderDocument;
 	return { ...options, url: positionals[0], output: positionals[1] };
 }
 
@@ -280,6 +244,71 @@ function parseArgs(args) {
 		}
 		result.options[optionKey] = optionValue;
 	});
+	Object.keys(OPTIONS_INFO).forEach(optionName => {
+		const optionInfo = getOptionInfo(optionName);
+		const optionKey = getOptionKey(optionName, optionInfo);
+		if (result.options[optionKey] === undefined && optionInfo.defaultValue !== undefined) {
+			result.options[optionKey] = OPTIONS_INFO[optionName].defaultValue;
+		}
+	});
+	result.options.acceptHeaders = {
+		font: result.options.acceptHeaderFont,
+		image: result.options.acceptHeaderImage,
+		stylesheet: result.options.acceptHeaderStylesheet,
+		script: result.options.acceptHeaderScript,
+		document: result.options.acceptHeaderDocument
+	};
+	if (result.options.browserArgs) {
+		const browserArguments = result.options.browserArguments || [];
+		browserArguments.push(...JSON.parse(result.options.browserArgs));
+		result.options.browserArgs = browserArguments;
+		delete result.options.browserArguments;
+	}
+	if (result.options.browserArguments) {
+		result.options.browserArgs = result.options.browserArguments;
+		delete result.options.browserArguments;
+	}
+	if (result.options.errorFile) {
+		result.options.errorsFile = result.options.errorFile;
+		delete result.options.errorFile;
+	}
+	if (result.options.errorTracesDisabled) {
+		result.options.errorsTracesDisabled = result.options.errorTracesDisabled;
+		delete result.options.errorTracesDisabled;
+	}
+	if (result.options.crawlReplaceUrls) {
+		result.options.crawlReplaceURLs = result.options.crawlReplaceUrls;
+		delete result.options.crawlReplaceUrls;
+	}
+	if (result.options.saveOriginalUrls) {
+		result.options.saveOriginalURLs = result.options.saveOriginalUrls;
+		delete result.options.saveOriginalUrls;
+	}
+	if (result.options.browserRemoteDebuggingUrl) {
+		result.options.browserRemoteDebuggingURL = result.options.browserRemoteDebuggingUrl;
+		delete result.options.browserRemoteDebuggingUrl;
+	}
+	if (result.options.crawlRemoveUrlFragment) {
+		result.options.crawlRemoveURLFragment = result.options.crawlRemoveUrlFragment;
+		delete result.options.crawlRemoveUrlFragment;
+	}
+	if (result.options.compressCss) {
+		result.options.compressCSS = result.options.compressCss;
+		delete result.options.compressCss;
+	}
+	if (result.options.compressHtml) {
+		result.options.compressHTML = result.options.compressHtml;
+		delete result.options.compressHtml;
+	}
+	if (result.options.insertMetaCsp) {
+		result.options.insertMetaCSP = result.options.insertMetaCsp;
+		delete result.options.insertMetaCsp;
+	}
+	delete result.options.acceptHeaderFont;
+	delete result.options.acceptHeaderImage;
+	delete result.options.acceptHeaderStylesheet;
+	delete result.options.acceptHeaderScript;
+	delete result.options.acceptHeaderDocument;
 	return result;
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "2.0.68",
+	"version": "2.0.69",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"engines": {

package/single-file-cli-api.js

@@ -101,7 +101,17 @@ async function initialize(options) {
 async function capture(urls, options) {
 	let newTasks;
 	const taskUrls = tasks.map(task => task.url);
-	newTasks = await Promise.all(urls.map(url => createTask(url, options)));
+	newTasks = await Promise.all(urls.map(value => {
+		let url, taskOptions;
+		if (Array.isArray(value)) {
+			url = value[0];
+			taskOptions = Object.assign({}, options, value[1]);
+		} else {
+			url = value;
+			taskOptions = options;
+		}
+		return createTask(url, taskOptions);
+	}));
 	newTasks = newTasks.filter(task => task && !taskUrls.includes(task.url));
 	if (newTasks.length) {
 		tasks = tasks.concat(newTasks);
@@ -127,7 +137,7 @@ async function finish(options) {
 					}
 				});
 				await writeTextFile(task.filename, pageContent);
-			} catch (error) {
+			} catch {
 				// ignored
 			}
 		}
@@ -137,7 +147,7 @@ async function finish(options) {
 	}
 }
 
-async function runTasks() {
+function runTasks() {
 	const availableTasks = tasks.filter(task => !task.status).length;
 	const processingTasks = tasks.filter(task => task.status == STATE_PROCESSING).length;
 	const promisesTasks = [];
@@ -151,7 +161,7 @@ async function runNextTask() {
 	const task = tasks.find(task => !task.status);
 	if (task) {
 		const options = task.options;
-		let taskOptions = JSON.parse(JSON.stringify(options));
+		const taskOptions = JSON.parse(JSON.stringify(options));
 		taskOptions.url = task.url;
 		task.status = STATE_PROCESSING;
 		await saveTasks();
@@ -166,6 +176,7 @@ async function runNextTask() {
 				newTasks = newTasks.filter(task => task &&
 					testMaxDepth(task) &&
 					!tasks.find(otherTask => otherTask.url == task.url) &&
+					!newTasks.find(otherTask => otherTask != task && otherTask.url == task.url) &&
 					(!options.crawlInnerLinksOnly || task.isInnerLink) &&
 					(!options.crawlNoParent || (task.isChild || !task.isInnerLink)));
 				tasks.splice(tasks.length, 0, ...newTasks);
@@ -282,7 +293,7 @@ async function capturePage(options) {
 			if (options.outputJson) {
 				filename += filename.endsWith(".json") ? "" : ".json";
 			}
-			const directoryName = await path.dirname(filename);
+			const directoryName = path.dirname(filename);
 			if (directoryName !== ".") {
 				await mkdir(directoryName, { recursive: true });
 			}
@@ -338,7 +349,7 @@ async function getFilename(filename, options, index = 1) {
 		if (options.filenameConflictAction != "skip") {
 			return getFilename(filename, options, index + 1);
 		}
-	} catch (_) {
+	} catch {
 		return newFilename;
 	}
 }

package/single-file-launcher.js

@@ -24,7 +24,7 @@
 import { initialize } from "./single-file-cli-api.js";
 import { closeBrowser } from "./lib/browser.js";
 import { Deno } from "./lib/deno-polyfill.js";
-import options from "./options.js";
+import { options, parseArgs } from "./options.js";
 
 const { readTextFile, readFile, exit, addSignalListener } = Deno;
 
@@ -54,7 +54,7 @@ async function run() {
 			delete options.settingsFile;
 		}
 		if (options.urlsFile) {
-			urls = (await readTextFile(options.urlsFile)).split("\n");
+			urls = await getUrlsFile(options.urlsFile);
 		} else {
 			urls = [options.url];
 		}
@@ -80,7 +80,7 @@ async function run() {
 			const cookiesContent = await readTextFile(options.browserCookiesFile);
 			try {
 				options.browserCookies = JSON.parse(cookiesContent);
-			} catch (error) {
+			} catch {
 				options.browserCookies = parseCookies(cookiesContent);
 			}
 		}
@@ -137,3 +137,56 @@ async function closeBrowserAndExit(code) {
 	await closeBrowser();
 	exit(code);
 }
+
+async function getUrlsFile(urlsFile) {
+	let urls = (await readTextFile(urlsFile)).split("\n");
+	urls = urls.map(value => {
+		value = value.trim();
+		let optionPosition = value.indexOf(" --");
+		if (optionPosition < 0) {
+			optionPosition = value.indexOf("\t--");
+		}
+		if (optionPosition > 0) {
+			const url = value.substring(0, optionPosition).trim();
+			const argsString = value.substring(optionPosition + 1).trim();
+			const args = [];
+			let previousCharacter, previousPreviousCharacter, lastQuoteCharacter;
+			let lastCharIndex = 0;
+			for (let currentCharIndex = 0; currentCharIndex < argsString.length; currentCharIndex++) {
+				const character = argsString[currentCharIndex];
+				if (character == lastQuoteCharacter && (previousCharacter != "\\" || previousPreviousCharacter == "\\")) {
+					args.push(argsString.substring(lastCharIndex, currentCharIndex));
+					lastQuoteCharacter = null;
+					lastCharIndex = currentCharIndex + 1;
+				} else if (!lastQuoteCharacter) {
+					if (character == "'" || character == "\"") {
+						lastQuoteCharacter = argsString[currentCharIndex];
+						lastCharIndex = currentCharIndex + 1;
+					} else {
+						const isSpaceCharacter = character == " " || character == "\t";
+						if (isSpaceCharacter || character == "=") {
+							if (isSpaceCharacter && (currentCharIndex == lastCharIndex + 1)) {
+								lastCharIndex++;
+							} else if (lastCharIndex < currentCharIndex) {
+								args.push(argsString.substring(lastCharIndex, currentCharIndex));
+								lastCharIndex = currentCharIndex + 1;
+							} else {
+								lastCharIndex = currentCharIndex + 1;
+							}
+						}
+					}
+				}
+				previousPreviousCharacter = previousCharacter;
+				previousCharacter = character;
+			}
+			if (lastCharIndex < argsString.length) {
+				args.push(argsString.substring(lastCharIndex).trim());
+			}
+			const { options } = parseArgs(args);
+			return [url, options];
+		} else {
+			return value;
+		}
+	});
+	return urls;
+}