single-file-cli 2.0.67 → 2.0.69

package/lib/single-file-script.js

@@ -41,7 +41,7 @@ function initSingleFile() {
 				xhrRequest.onreadystatechange = () => {
 					if (xhrRequest.readyState == XMLHttpRequest.DONE) {
 						resolve({
-							arrayBuffer: async () => xhrRequest.response || new ArrayBuffer(),
+							arrayBuffer: () => xhrRequest.response || new ArrayBuffer(),
 							headers: { get: headerName => xhrRequest.getResponseHeader(headerName) },
 							status: xhrRequest.status
 						});

package/lib/version.js

@@ -1 +1 @@
-export const version = "2.0.67";
+export const version = "2.0.69";

package/options.js

@@ -50,6 +50,7 @@ const OPTIONS_INFO = {
 	"browser-load-max-time": { description: "Maximum delay of time to wait for page loading in ms", type: "number", defaultValue: 60000 },
 	"browser-wait-delay": { description: "Time to wait before capturing the page in ms", type: "number" },
 	"browser-wait-until": { description: "When to consider the page is loaded (InteractiveTime, networkIdle, networkAlmostIdle, load, domContentLoaded)", type: "string", defaultValue: "networkIdle" },
+	"browser-wait-until-delay": { description: "Delay of time in ms to wait before considering the page is loaded when the value of --browser-wait-until is reached", type: "number", defaultValue: 1000 },
 	"browser-wait-until-fallback": { description: "Retry with the next value of --browser-wait-until when a timeout error is thrown", type: "boolean", defaultValue: true },
 	"browser-debug": { description: "Enable debug mode", type: "boolean" },
 	"browser-script": { description: "Path of a script executed in the page (and all the frames) before it is loaded", type: "string[]" },
@@ -121,7 +122,7 @@ const OPTIONS_INFO = {
 	"remove-alternative-images": { description: "Remove images for alternative sizes of screen", type: "boolean", defaultValue: true },
 	"save-original-URLs": { description: "Save the original URLS in the embedded contents", type: "boolean" },
 	"save-raw-page": { description: "Save the original page without interpreting it into the browser", type: "boolean" },
-	"urls-file": { description: "Path to a text file containing a list of URLs (separated by a newline) to save", type: "string" },
+	"urls-file": { description: "Path to a text file containing a list of URLs (separated by a newline) to save. You can also pass the options after each URL (e.g. 'https://www.example.com --filename-template={page-title}.html')", type: "string" },
 	"user-agent": { description: "User-agent of the browser", type: "string" },
 	"accept-language": { description: "Accept language of the browser", type: "string" },
 	"platform": { description: "Platform of the browser (default value is \"Android\" when using --browser-mobile-emulation)", type: "string" },
@@ -149,10 +150,10 @@ const OPTIONS_INFO = {
 };
 
 const { args, exit } = Deno;
-const options = await getOptions();
-export default options;
+const options = getOptions();
+export { options, parseArgs };
 
-async function getOptions() {
+function getOptions() {
 	const { positionals, options } = parseArgs(Array.from(args));
 	const unknownOptions = [];
 	positionals.forEach(positional => {
@@ -185,43 +186,6 @@ async function getOptions() {
 		console.log(version); // eslint-disable-line no-console
 		exit(0);
 	}
-	Object.keys(OPTIONS_INFO).forEach(optionName => {
-		const optionInfo = getOptionInfo(optionName);
-		const optionKey = getOptionKey(optionName, optionInfo);
-		if (options[optionKey] === undefined && optionInfo.defaultValue !== undefined) {
-			options[optionKey] = OPTIONS_INFO[optionName].defaultValue;
-		}
-	});
-	options.acceptHeaders = {
-		font: options.acceptHeaderFont,
-		image: options.acceptHeaderImage,
-		stylesheet: options.acceptHeaderStylesheet,
-		script: options.acceptHeaderScript,
-		document: options.acceptHeaderDocument
-	};
-	if (options.browserArgs) {
-		const browserArguments = options.browserArguments || [];
-		browserArguments.push(...JSON.parse(options.browserArgs));
-		options.browserArgs = browserArguments;
-		delete options.browserArguments;
-	}
-	if (options.browserArguments) {
-		options.browserArgs = options.browserArguments;
-		delete options.browserArguments;
-	}
-	if (options.errorFile) {
-		options.errorsFile = options.errorFile;
-		delete options.errorFile;
-	}
-	if (options.errorTracesDisabled) {
-		options.errorsTracesDisabled = options.errorTracesDisabled;
-		delete options.errorTracesDisabled;
-	}
-	delete options.acceptHeaderFont;
-	delete options.acceptHeaderImage;
-	delete options.acceptHeaderStylesheet;
-	delete options.acceptHeaderScript;
-	delete options.acceptHeaderDocument;
 	return { ...options, url: positionals[0], output: positionals[1] };
 }
 
@@ -280,6 +244,71 @@ function parseArgs(args) {
 		}
 		result.options[optionKey] = optionValue;
 	});
+	Object.keys(OPTIONS_INFO).forEach(optionName => {
+		const optionInfo = getOptionInfo(optionName);
+		const optionKey = getOptionKey(optionName, optionInfo);
+		if (result.options[optionKey] === undefined && optionInfo.defaultValue !== undefined) {
+			result.options[optionKey] = OPTIONS_INFO[optionName].defaultValue;
+		}
+	});
+	result.options.acceptHeaders = {
+		font: result.options.acceptHeaderFont,
+		image: result.options.acceptHeaderImage,
+		stylesheet: result.options.acceptHeaderStylesheet,
+		script: result.options.acceptHeaderScript,
+		document: result.options.acceptHeaderDocument
+	};
+	if (result.options.browserArgs) {
+		const browserArguments = result.options.browserArguments || [];
+		browserArguments.push(...JSON.parse(result.options.browserArgs));
+		result.options.browserArgs = browserArguments;
+		delete result.options.browserArguments;
+	}
+	if (result.options.browserArguments) {
+		result.options.browserArgs = result.options.browserArguments;
+		delete result.options.browserArguments;
+	}
+	if (result.options.errorFile) {
+		result.options.errorsFile = result.options.errorFile;
+		delete result.options.errorFile;
+	}
+	if (result.options.errorTracesDisabled) {
+		result.options.errorsTracesDisabled = result.options.errorTracesDisabled;
+		delete result.options.errorTracesDisabled;
+	}
+	if (result.options.crawlReplaceUrls) {
+		result.options.crawlReplaceURLs = result.options.crawlReplaceUrls;
+		delete result.options.crawlReplaceUrls;
+	}
+	if (result.options.saveOriginalUrls) {
+		result.options.saveOriginalURLs = result.options.saveOriginalUrls;
+		delete result.options.saveOriginalUrls;
+	}
+	if (result.options.browserRemoteDebuggingUrl) {
+		result.options.browserRemoteDebuggingURL = result.options.browserRemoteDebuggingUrl;
+		delete result.options.browserRemoteDebuggingUrl;
+	}
+	if (result.options.crawlRemoveUrlFragment) {
+		result.options.crawlRemoveURLFragment = result.options.crawlRemoveUrlFragment;
+		delete result.options.crawlRemoveUrlFragment;
+	}
+	if (result.options.compressCss) {
+		result.options.compressCSS = result.options.compressCss;
+		delete result.options.compressCss;
+	}
+	if (result.options.compressHtml) {
+		result.options.compressHTML = result.options.compressHtml;
+		delete result.options.compressHtml;
+	}
+	if (result.options.insertMetaCsp) {
+		result.options.insertMetaCSP = result.options.insertMetaCsp;
+		delete result.options.insertMetaCsp;
+	}
+	delete result.options.acceptHeaderFont;
+	delete result.options.acceptHeaderImage;
+	delete result.options.acceptHeaderStylesheet;
+	delete result.options.acceptHeaderScript;
+	delete result.options.acceptHeaderDocument;
 	return result;
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "2.0.67",
+	"version": "2.0.69",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"engines": {

package/single-file-cli-api.js

@@ -101,7 +101,17 @@ async function initialize(options) {
 async function capture(urls, options) {
 	let newTasks;
 	const taskUrls = tasks.map(task => task.url);
-	newTasks = await Promise.all(urls.map(url => createTask(url, options)));
+	newTasks = await Promise.all(urls.map(value => {
+		let url, taskOptions;
+		if (Array.isArray(value)) {
+			url = value[0];
+			taskOptions = Object.assign({}, options, value[1]);
+		} else {
+			url = value;
+			taskOptions = options;
+		}
+		return createTask(url, taskOptions);
+	}));
 	newTasks = newTasks.filter(task => task && !taskUrls.includes(task.url));
 	if (newTasks.length) {
 		tasks = tasks.concat(newTasks);
@@ -127,7 +137,7 @@ async function finish(options) {
 					}
 				});
 				await writeTextFile(task.filename, pageContent);
-			} catch (error) {
+			} catch {
 				// ignored
 			}
 		}
@@ -137,7 +147,7 @@ async function finish(options) {
 	}
 }
 
-async function runTasks() {
+function runTasks() {
 	const availableTasks = tasks.filter(task => !task.status).length;
 	const processingTasks = tasks.filter(task => task.status == STATE_PROCESSING).length;
 	const promisesTasks = [];
@@ -151,7 +161,7 @@ async function runNextTask() {
 	const task = tasks.find(task => !task.status);
 	if (task) {
 		const options = task.options;
-		let taskOptions = JSON.parse(JSON.stringify(options));
+		const taskOptions = JSON.parse(JSON.stringify(options));
 		taskOptions.url = task.url;
 		task.status = STATE_PROCESSING;
 		await saveTasks();
@@ -162,10 +172,11 @@ async function runNextTask() {
 			task.filename = pageData.filename;
 			if (options.crawlLinks && testMaxDepth(task)) {
 				const urls = pageData.links;
-				let newTasks = await Promise.all(urls.map(url => createTask(url, options, task, tasks[0])));
+				let newTasks = await Promise.all(urls.map(url => createTask(url, options, task, task.rootTaskURL || task.url)));
 				newTasks = newTasks.filter(task => task &&
 					testMaxDepth(task) &&
 					!tasks.find(otherTask => otherTask.url == task.url) &&
+					!newTasks.find(otherTask => otherTask != task && otherTask.url == task.url) &&
 					(!options.crawlInnerLinksOnly || task.isInnerLink) &&
 					(!options.crawlNoParent || (task.isChild || !task.isInnerLink)));
 				tasks.splice(tasks.length, 0, ...newTasks);
@@ -182,7 +193,7 @@ function testMaxDepth(task) {
 		(options.crawlExternalLinksMaxDepth == 0 || task.externalLinkDepth < options.crawlExternalLinksMaxDepth);
 }
 
-async function createTask(url, options, parentTask, rootTask) {
+async function createTask(url, options, parentTask, rootTaskURL) {
 	url = parentTask ? rewriteURL(url, options.crawlRemoveURLFragment, options.crawlRewriteRules) : url;
 	if (url) {
 		if (!VALID_URL_TEST.test(url)) {
@@ -195,15 +206,15 @@ async function createTask(url, options, parentTask, rootTask) {
 				throw new Error("Invalid URL or file path: " + url, { cause: error });
 			}
 		}
-		const isInnerLink = rootTask && url.startsWith(getHostURL(rootTask.url));
-		const rootBaseURIMatch = rootTask && rootTask.url.match(/(.*?)[^/]*$/);
+		const isInnerLink = rootTaskURL && url.startsWith(getHostURL(rootTaskURL));
+		const rootBaseURIMatch = rootTaskURL && rootTaskURL.match(/(.*?)[^/]*$/);
 		const isChild = isInnerLink && rootBaseURIMatch && rootBaseURIMatch[1] && url.startsWith(rootBaseURIMatch[1]);
 		return {
 			url,
 			isInnerLink,
 			isChild,
 			originalUrl: url,
-			rootBaseURI: rootBaseURIMatch && rootBaseURIMatch[1],
+			rootTaskURL,
 			depth: parentTask ? parentTask.depth + 1 : 0,
 			externalLinkDepth: isInnerLink ? -1 : parentTask ? parentTask.externalLinkDepth + 1 : -1,
 			options
@@ -282,7 +293,7 @@ async function capturePage(options) {
 			if (options.outputJson) {
 				filename += filename.endsWith(".json") ? "" : ".json";
 			}
-			const directoryName = await path.dirname(filename);
+			const directoryName = path.dirname(filename);
 			if (directoryName !== ".") {
 				await mkdir(directoryName, { recursive: true });
 			}
@@ -338,7 +349,7 @@ async function getFilename(filename, options, index = 1) {
 		if (options.filenameConflictAction != "skip") {
 			return getFilename(filename, options, index + 1);
 		}
-	} catch (_) {
+	} catch {
 		return newFilename;
 	}
 }

package/single-file-launcher.js

@@ -24,7 +24,7 @@
 import { initialize } from "./single-file-cli-api.js";
 import { closeBrowser } from "./lib/browser.js";
 import { Deno } from "./lib/deno-polyfill.js";
-import options from "./options.js";
+import { options, parseArgs } from "./options.js";
 
 const { readTextFile, readFile, exit, addSignalListener } = Deno;
 
@@ -54,7 +54,7 @@ async function run() {
 			delete options.settingsFile;
 		}
 		if (options.urlsFile) {
-			urls = (await readTextFile(options.urlsFile)).split("\n");
+			urls = await getUrlsFile(options.urlsFile);
 		} else {
 			urls = [options.url];
 		}
@@ -80,7 +80,7 @@ async function run() {
 			const cookiesContent = await readTextFile(options.browserCookiesFile);
 			try {
 				options.browserCookies = JSON.parse(cookiesContent);
-			} catch (error) {
+			} catch {
 				options.browserCookies = parseCookies(cookiesContent);
 			}
 		}
@@ -137,3 +137,56 @@ async function closeBrowserAndExit(code) {
 	await closeBrowser();
 	exit(code);
 }
+
+async function getUrlsFile(urlsFile) {
+	let urls = (await readTextFile(urlsFile)).split("\n");
+	urls = urls.map(value => {
+		value = value.trim();
+		let optionPosition = value.indexOf(" --");
+		if (optionPosition < 0) {
+			optionPosition = value.indexOf("\t--");
+		}
+		if (optionPosition > 0) {
+			const url = value.substring(0, optionPosition).trim();
+			const argsString = value.substring(optionPosition + 1).trim();
+			const args = [];
+			let previousCharacter, previousPreviousCharacter, lastQuoteCharacter;
+			let lastCharIndex = 0;
+			for (let currentCharIndex = 0; currentCharIndex < argsString.length; currentCharIndex++) {
+				const character = argsString[currentCharIndex];
+				if (character == lastQuoteCharacter && (previousCharacter != "\\" || previousPreviousCharacter == "\\")) {
+					args.push(argsString.substring(lastCharIndex, currentCharIndex));
+					lastQuoteCharacter = null;
+					lastCharIndex = currentCharIndex + 1;
+				} else if (!lastQuoteCharacter) {
+					if (character == "'" || character == "\"") {
+						lastQuoteCharacter = argsString[currentCharIndex];
+						lastCharIndex = currentCharIndex + 1;
+					} else {
+						const isSpaceCharacter = character == " " || character == "\t";
+						if (isSpaceCharacter || character == "=") {
+							if (isSpaceCharacter && (currentCharIndex == lastCharIndex + 1)) {
+								lastCharIndex++;
+							} else if (lastCharIndex < currentCharIndex) {
+								args.push(argsString.substring(lastCharIndex, currentCharIndex));
+								lastCharIndex = currentCharIndex + 1;
+							} else {
+								lastCharIndex = currentCharIndex + 1;
+							}
+						}
+					}
+				}
+				previousPreviousCharacter = previousCharacter;
+				previousCharacter = character;
+			}
+			if (lastCharIndex < argsString.length) {
+				args.push(argsString.substring(lastCharIndex).trim());
+			}
+			const { options } = parseArgs(args);
+			return [url, options];
+		} else {
+			return value;
+		}
+	});
+	return urls;
+}