single-file-cli 2.0.68 → 2.0.70

package/lib/single-file-script.js

@@ -41,7 +41,7 @@ function initSingleFile() {
 				xhrRequest.onreadystatechange = () => {
 					if (xhrRequest.readyState == XMLHttpRequest.DONE) {
 						resolve({
-							arrayBuffer: async () => xhrRequest.response || new ArrayBuffer(),
+							arrayBuffer: () => xhrRequest.response || new ArrayBuffer(),
 							headers: { get: headerName => xhrRequest.getResponseHeader(headerName) },
 							status: xhrRequest.status
 						});

package/lib/version.js

@@ -1 +1 @@
-export const version = "2.0.68";
+export const version = "2.0.70";

package/options.js

@@ -42,6 +42,7 @@ const OPTIONS_INFO = {
 	"block-scripts": { description: "Block scripts", type: "boolean", defaultValue: true },
 	"block-videos": { description: "Block videos", type: "boolean", defaultValue: true },
 	"block-mixed-content": { description: "Block mixed contents", type: "boolean" },
+	"blocked-URL-pattern": { description: "Regular expression matching URLs to block (e.g. 'annoying-banners\\.com')", type: "string[]" },
 	"browser-server": { description: "Server to connect to", type: "string" },
 	"browser-headless": { description: "Run the browser in headless mode", type: "boolean", defaultValue: true },
 	"browser-executable-path": { description: "Path to chrome/chromium executable", type: "string" },
@@ -50,6 +51,7 @@ const OPTIONS_INFO = {
 	"browser-load-max-time": { description: "Maximum delay of time to wait for page loading in ms", type: "number", defaultValue: 60000 },
 	"browser-wait-delay": { description: "Time to wait before capturing the page in ms", type: "number" },
 	"browser-wait-until": { description: "When to consider the page is loaded (InteractiveTime, networkIdle, networkAlmostIdle, load, domContentLoaded)", type: "string", defaultValue: "networkIdle" },
+	"browser-wait-until-delay": { description: "Delay of time in ms to wait before considering the page is loaded when the value of --browser-wait-until is reached", type: "number", defaultValue: 1000 },
 	"browser-wait-until-fallback": { description: "Retry with the next value of --browser-wait-until when a timeout error is thrown", type: "boolean", defaultValue: true },
 	"browser-debug": { description: "Enable debug mode", type: "boolean" },
 	"browser-script": { description: "Path of a script executed in the page (and all the frames) before it is loaded", type: "string[]" },
@@ -66,6 +68,7 @@ const OPTIONS_INFO = {
 	"browser-device-height": { description: "Height of the device viewport in pixels (default value is 800 when using --browser-mobile-emulation)", type: "number" },
 	"browser-device-scale-factor": { description: "Scale factor of the device viewport (default value is 2 when using --browser-mobile-emulation)", type: "number" },
 	"console-messages-file": { description: "Path of the file where to save the console messages in JSON format", type: "string" },
+	"debug-messages-file": { description: "Path of the file where to save the debug messages", type: "string" },
 	"compress-CSS": { description: "Compress CSS stylesheets", type: "boolean" },
 	"compress-HTML": { description: "Compress HTML content", type: "boolean", defaultValue: true },
 	"crawl-links": { description: "Crawl and save pages found via inner links", type: "boolean" },
@@ -121,7 +124,7 @@ const OPTIONS_INFO = {
 	"remove-alternative-images": { description: "Remove images for alternative sizes of screen", type: "boolean", defaultValue: true },
 	"save-original-URLs": { description: "Save the original URLS in the embedded contents", type: "boolean" },
 	"save-raw-page": { description: "Save the original page without interpreting it into the browser", type: "boolean" },
-	"urls-file": { description: "Path to a text file containing a list of URLs (separated by a newline) to save", type: "string" },
+	"urls-file": { description: "Path to a text file containing a list of URLs (separated by a newline) to save. You can also pass the options after each URL (e.g. 'https://www.example.com --filename-template={page-title}.html')", type: "string" },
 	"user-agent": { description: "User-agent of the browser", type: "string" },
 	"accept-language": { description: "Accept language of the browser", type: "string" },
 	"platform": { description: "Platform of the browser (default value is \"Android\" when using --browser-mobile-emulation)", type: "string" },
@@ -145,14 +148,14 @@ const OPTIONS_INFO = {
 	"resolve-links": { description: "Resolve link URLs to absolute URLs", type: "boolean", defaultValue: true },
 	"settings-file": { description: "Path to a JSON file containing the settings exported from the web extension", type: "string" },
 	"settings-file-profile": { description: "Name of the profile to use when using --settings-file", type: "string", defaultValue: "default" },
-	"group-duplicate-stylesheets": { description: "Group duplicate inline stylesheets into a single stylesheet in order to reduce the size of the page", type: "boolean", defaultValue: true }
+	"group-duplicate-stylesheets": { description: "Group duplicate inline stylesheets into a single stylesheet in order to reduce the size of the page", type: "boolean", defaultValue: false }
 };
 
 const { args, exit } = Deno;
-const options = await getOptions();
-export default options;
+const options = getOptions();
+export { options, parseArgs };
 
-async function getOptions() {
+function getOptions() {
 	const { positionals, options } = parseArgs(Array.from(args));
 	const unknownOptions = [];
 	positionals.forEach(positional => {
@@ -185,47 +188,10 @@ async function getOptions() {
 		console.log(version); // eslint-disable-line no-console
 		exit(0);
 	}
-	Object.keys(OPTIONS_INFO).forEach(optionName => {
-		const optionInfo = getOptionInfo(optionName);
-		const optionKey = getOptionKey(optionName, optionInfo);
-		if (options[optionKey] === undefined && optionInfo.defaultValue !== undefined) {
-			options[optionKey] = OPTIONS_INFO[optionName].defaultValue;
-		}
-	});
-	options.acceptHeaders = {
-		font: options.acceptHeaderFont,
-		image: options.acceptHeaderImage,
-		stylesheet: options.acceptHeaderStylesheet,
-		script: options.acceptHeaderScript,
-		document: options.acceptHeaderDocument
-	};
-	if (options.browserArgs) {
-		const browserArguments = options.browserArguments || [];
-		browserArguments.push(...JSON.parse(options.browserArgs));
-		options.browserArgs = browserArguments;
-		delete options.browserArguments;
-	}
-	if (options.browserArguments) {
-		options.browserArgs = options.browserArguments;
-		delete options.browserArguments;
-	}
-	if (options.errorFile) {
-		options.errorsFile = options.errorFile;
-		delete options.errorFile;
-	}
-	if (options.errorTracesDisabled) {
-		options.errorsTracesDisabled = options.errorTracesDisabled;
-		delete options.errorTracesDisabled;
-	}
-	delete options.acceptHeaderFont;
-	delete options.acceptHeaderImage;
-	delete options.acceptHeaderStylesheet;
-	delete options.acceptHeaderScript;
-	delete options.acceptHeaderDocument;
 	return { ...options, url: positionals[0], output: positionals[1] };
 }
 
-function parseArgs(args) {
+function parseArgs(args, setDefaultValues = true) {
 	const positionals = [];
 	const options = {};
 	const result = { positionals, options: {} };
@@ -280,6 +246,83 @@ function parseArgs(args) {
 		}
 		result.options[optionKey] = optionValue;
 	});
+	if (setDefaultValues) {
+		Object.keys(OPTIONS_INFO).forEach(optionName => {
+			const optionInfo = getOptionInfo(optionName);
+			const optionKey = getOptionKey(optionName, optionInfo);
+			if (result.options[optionKey] === undefined && optionInfo.defaultValue !== undefined) {
+				result.options[optionKey] = OPTIONS_INFO[optionName].defaultValue;
+			}
+		});
+	}
+	if (result.options.acceptHeaderFont ||
+		result.options.acceptHeaderImage ||
+		result.options.acceptHeaderStylesheet ||
+		result.options.acceptHeaderScript ||
+		result.options.acceptHeaderDocument) {
+		result.options.acceptHeaders = {
+			font: result.options.acceptHeaderFont,
+			image: result.options.acceptHeaderImage,
+			stylesheet: result.options.acceptHeaderStylesheet,
+			script: result.options.acceptHeaderScript,
+			document: result.options.acceptHeaderDocument
+		};
+		delete result.options.acceptHeaderFont;
+		delete result.options.acceptHeaderImage;
+		delete result.options.acceptHeaderStylesheet;
+		delete result.options.acceptHeaderScript;
+		delete result.options.acceptHeaderDocument;
+	}
+	if (result.options.browserArgs) {
+		const browserArguments = result.options.browserArguments || [];
+		browserArguments.push(...JSON.parse(result.options.browserArgs));
+		result.options.browserArgs = browserArguments;
+		delete result.options.browserArguments;
+	}
+	if (result.options.browserArguments) {
+		result.options.browserArgs = result.options.browserArguments;
+		delete result.options.browserArguments;
+	}
+	if (result.options.errorFile) {
+		result.options.errorsFile = result.options.errorFile;
+		delete result.options.errorFile;
+	}
+	if (result.options.errorTracesDisabled) {
+		result.options.errorsTracesDisabled = result.options.errorTracesDisabled;
+		delete result.options.errorTracesDisabled;
+	}
+	if (result.options.crawlReplaceUrls) {
+		result.options.crawlReplaceURLs = result.options.crawlReplaceUrls;
+		delete result.options.crawlReplaceUrls;
+	}
+	if (result.options.saveOriginalUrls) {
+		result.options.saveOriginalURLs = result.options.saveOriginalUrls;
+		delete result.options.saveOriginalUrls;
+	}
+	if (result.options.browserRemoteDebuggingUrl) {
+		result.options.browserRemoteDebuggingURL = result.options.browserRemoteDebuggingUrl;
+		delete result.options.browserRemoteDebuggingUrl;
+	}
+	if (result.options.crawlRemoveUrlFragment) {
+		result.options.crawlRemoveURLFragment = result.options.crawlRemoveUrlFragment;
+		delete result.options.crawlRemoveUrlFragment;
+	}
+	if (result.options.compressCss) {
+		result.options.compressCSS = result.options.compressCss;
+		delete result.options.compressCss;
+	}
+	if (result.options.compressHtml) {
+		result.options.compressHTML = result.options.compressHtml;
+		delete result.options.compressHtml;
+	}
+	if (result.options.insertMetaCsp) {
+		result.options.insertMetaCSP = result.options.insertMetaCsp;
+		delete result.options.insertMetaCsp;
+	}
+	if (result.options.blockedUrlPatterns) {
+		result.options.blockedURLPatterns = result.options.blockedUrlPatterns;
+		delete result.options.blockedUrlPatterns;
+	}
 	return result;
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "2.0.68",
+	"version": "2.0.70",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"engines": {

package/single-file-cli-api.js

@@ -39,8 +39,9 @@ const DEFAULT_OPTIONS = {
 	filenameTemplate: "{page-title} ({date-locale} {time-locale}).html",
 	filenameMaxLength: 192,
 	filenameMaxLengthUnit: "bytes",
-	filenameReplacedCharacters: ["~", "+", "\\\\", "?", "%", "*", ":", "|", "\"", "<", ">", "\x00-\x1f", "\x7F"],
+	filenameReplacedCharacters: ["~", "+", "?", "%", "*", ":", "|", "\"", "<", ">", "\\\\", "\x00-\x1f", "\x7F"],
 	filenameReplacementCharacter: "_",
+	filenameReplacementCharacters: ["～", "＋", "？", "％", "＊", "：", "｜", "＂", "＜", "＞", "＼"],
 	maxResourceSize: 10,
 	backgroundSave: true,
 	removeAlternativeFonts: true,
@@ -101,7 +102,17 @@ async function initialize(options) {
 async function capture(urls, options) {
 	let newTasks;
 	const taskUrls = tasks.map(task => task.url);
-	newTasks = await Promise.all(urls.map(url => createTask(url, options)));
+	newTasks = await Promise.all(urls.map(value => {
+		let url, taskOptions;
+		if (Array.isArray(value)) {
+			url = value[0];
+			taskOptions = Object.assign({}, options, value[1]);
+		} else {
+			url = value;
+			taskOptions = options;
+		}
+		return createTask(url, taskOptions);
+	}));
 	newTasks = newTasks.filter(task => task && !taskUrls.includes(task.url));
 	if (newTasks.length) {
 		tasks = tasks.concat(newTasks);
@@ -127,7 +138,7 @@ async function finish(options) {
 					}
 				});
 				await writeTextFile(task.filename, pageContent);
-			} catch (error) {
+			} catch {
 				// ignored
 			}
 		}
@@ -137,7 +148,7 @@ async function finish(options) {
 	}
 }
 
-async function runTasks() {
+function runTasks() {
 	const availableTasks = tasks.filter(task => !task.status).length;
 	const processingTasks = tasks.filter(task => task.status == STATE_PROCESSING).length;
 	const promisesTasks = [];
@@ -151,7 +162,7 @@ async function runNextTask() {
 	const task = tasks.find(task => !task.status);
 	if (task) {
 		const options = task.options;
-		let taskOptions = JSON.parse(JSON.stringify(options));
+		const taskOptions = JSON.parse(JSON.stringify(options));
 		taskOptions.url = task.url;
 		task.status = STATE_PROCESSING;
 		await saveTasks();
@@ -166,6 +177,7 @@ async function runNextTask() {
 				newTasks = newTasks.filter(task => task &&
 					testMaxDepth(task) &&
 					!tasks.find(otherTask => otherTask.url == task.url) &&
+					!newTasks.find(otherTask => otherTask != task && otherTask.url == task.url) &&
 					(!options.crawlInnerLinksOnly || task.isInnerLink) &&
 					(!options.crawlNoParent || (task.isChild || !task.isInnerLink)));
 				tasks.splice(tasks.length, 0, ...newTasks);
@@ -251,6 +263,10 @@ async function capturePage(options) {
 		if (options.consoleMessagesFile && pageData.consoleMessages) {
 			await writeTextFile(options.consoleMessagesFile, JSON.stringify(pageData.consoleMessages, null, 2));
 		}
+		if (options.debugMessagesFile && pageData.debugMessages) {
+			await writeTextFile(options.debugMessagesFile, pageData.debugMessages.map(([timestamp, message]) =>
+				`[${new Date(timestamp).toISOString()}] ${message.join(" ")}`).join("\n"));
+		}
 		if (options.outputJson) {
 			if (content instanceof Uint8Array) {
 				const fileReader = new FileReader();
@@ -282,7 +298,7 @@ async function capturePage(options) {
 			if (options.outputJson) {
 				filename += filename.endsWith(".json") ? "" : ".json";
 			}
-			const directoryName = await path.dirname(filename);
+			const directoryName = path.dirname(filename);
 			if (directoryName !== ".") {
 				await mkdir(directoryName, { recursive: true });
 			}
@@ -305,6 +321,13 @@ async function capturePage(options) {
 		} else {
 			console.error(error.message || error, message); // eslint-disable-line no-console
 		}
+		if (options.consoleMessagesFile && error.consoleMessages) {
+			await writeTextFile(options.consoleMessagesFile, JSON.stringify(error.consoleMessages, null, 2));
+		}
+		if (options.debugMessagesFile && error.debugMessages) {
+			await writeTextFile(options.debugMessagesFile, error.debugMessages.map(([timestamp, message]) =>
+				`[${new Date(timestamp).toISOString()}] ${message.join(" ")}`).join("\n"));
+		}
 	}
 }
 
@@ -338,7 +361,7 @@ async function getFilename(filename, options, index = 1) {
 		if (options.filenameConflictAction != "skip") {
 			return getFilename(filename, options, index + 1);
 		}
-	} catch (_) {
+	} catch {
 		return newFilename;
 	}
 }

package/single-file-launcher.js

@@ -24,7 +24,7 @@
 import { initialize } from "./single-file-cli-api.js";
 import { closeBrowser } from "./lib/browser.js";
 import { Deno } from "./lib/deno-polyfill.js";
-import options from "./options.js";
+import { options, parseArgs } from "./options.js";
 
 const { readTextFile, readFile, exit, addSignalListener } = Deno;
 
@@ -54,7 +54,7 @@ async function run() {
 			delete options.settingsFile;
 		}
 		if (options.urlsFile) {
-			urls = (await readTextFile(options.urlsFile)).split("\n");
+			urls = await getUrlsFile(options.urlsFile);
 		} else {
 			urls = [options.url];
 		}
@@ -80,7 +80,7 @@ async function run() {
 			const cookiesContent = await readTextFile(options.browserCookiesFile);
 			try {
 				options.browserCookies = JSON.parse(cookiesContent);
-			} catch (error) {
+			} catch {
 				options.browserCookies = parseCookies(cookiesContent);
 			}
 		}
@@ -137,3 +137,56 @@ async function closeBrowserAndExit(code) {
 	await closeBrowser();
 	exit(code);
 }
+
+async function getUrlsFile(urlsFile) {
+	let urls = (await readTextFile(urlsFile)).split("\n");
+	urls = urls.map(value => {
+		value = value.trim();
+		let optionPosition = value.indexOf(" --");
+		if (optionPosition < 0) {
+			optionPosition = value.indexOf("\t--");
+		}
+		if (optionPosition > 0) {
+			const url = value.substring(0, optionPosition).trim();
+			const argsString = value.substring(optionPosition + 1).trim();
+			const args = [];
+			let previousCharacter, previousPreviousCharacter, lastQuoteCharacter;
+			let lastCharIndex = 0;
+			for (let currentCharIndex = 0; currentCharIndex < argsString.length; currentCharIndex++) {
+				const character = argsString[currentCharIndex];
+				if (character == lastQuoteCharacter && (previousCharacter != "\\" || previousPreviousCharacter == "\\")) {
+					args.push(argsString.substring(lastCharIndex, currentCharIndex));
+					lastQuoteCharacter = null;
+					lastCharIndex = currentCharIndex + 1;
+				} else if (!lastQuoteCharacter) {
+					if (character == "'" || character == "\"") {
+						lastQuoteCharacter = argsString[currentCharIndex];
+						lastCharIndex = currentCharIndex + 1;
+					} else {
+						const isSpaceCharacter = character == " " || character == "\t";
+						if (isSpaceCharacter || character == "=") {
+							if (isSpaceCharacter && (currentCharIndex == lastCharIndex + 1)) {
+								lastCharIndex++;
+							} else if (lastCharIndex < currentCharIndex) {
+								args.push(argsString.substring(lastCharIndex, currentCharIndex));
+								lastCharIndex = currentCharIndex + 1;
+							} else {
+								lastCharIndex = currentCharIndex + 1;
+							}
+						}
+					}
+				}
+				previousPreviousCharacter = previousCharacter;
+				previousCharacter = character;
+			}
+			if (lastCharIndex < argsString.length) {
+				args.push(argsString.substring(lastCharIndex).trim());
+			}
+			const { options } = parseArgs(args, false);
+			return [url, options];
+		} else {
+			return value;
+		}
+	});
+	return urls;
+}