single-file-cli 1.0.0

package/package.json

@@ -0,0 +1,37 @@
+{
+	"name": "single-file-cli",
+	"version": "1.0.0",
+	"description": "SingleFile CLI",
+	"author": "Gildas Lormeau",
+	"license": "AGPL-3.0-or-later",
+	"main": "single-file-cli-api.js",
+	"bin": {
+		"single-file": "./single-file"
+	},
+	"scripts": {
+		"build": "./build-lib.sh"
+	},
+	"repository": {
+	  "type": "git",
+	  "url": "git+https://github.com/gildas-lormeau/single-file-cli.git"
+	},
+	"bugs": {
+	  "url": "https://github.com/gildas-lormeau/single-file-cli/issues"
+	},
+	"homepage": "https://github.com/gildas-lormeau/single-file-cli#readme",
+	"dependencies": {
+		"file-url": "3.0.0",
+		"iconv-lite": "0.6.3",
+		"jsdom": "19.0.0",
+		"puppeteer-core": "13.5.2",
+		"selenium-webdriver": "4.1.1",
+		"single-file-core": "1.0.6",
+		"strong-data-uri": "1.0.6",
+		"yargs": "17.4.0"
+	},
+	"devDependencies": {
+		"@rollup/plugin-node-resolve": "13.3.0",
+		"rollup": "2.60.0",
+		"rollup-plugin-terser": "7.0.2"
+	}
+}

package/rollup.config.dev.js

@@ -0,0 +1,64 @@
+import resolve from "@rollup/plugin-node-resolve";
+import { terser } from "rollup-plugin-terser";
+
+const PLUGINS = [resolve({ moduleDirectories: [".."] })];
+const EXTERNAL = ["single-file-core"];
+
+export default [{
+	input: ["single-file-core/index.js"],
+	output: [{
+		file: "lib/single-file.js",
+		format: "umd",
+		name: "singlefile",
+		plugins: []
+	}],
+	plugins: PLUGINS,
+	external: EXTERNAL
+}, {
+	input: ["single-file-core/processors/frame-tree/content/content-frame-tree.js"],
+	output: [{
+		file: "lib/single-file-frames.js",
+		format: "umd",
+		name: "singlefile",
+		plugins: []
+	}],
+	plugins: PLUGINS,
+	external: EXTERNAL
+}, {
+	input: ["single-file-core/single-file-bootstrap.js"],
+	output: [{
+		file: "lib/single-file-bootstrap.js",
+		format: "umd",
+		name: "singlefileBootstrap",
+		plugins: []
+	}],
+	plugins: PLUGINS,
+	external: EXTERNAL
+}, {
+	input: ["single-file-core/processors/hooks/content/content-hooks-web.js"],
+	output: [{
+		file: "lib/single-file-hooks.js",
+		format: "iife",
+		plugins: []
+	}],
+	plugins: PLUGINS,
+	external: EXTERNAL
+}, {
+	input: ["single-file-core/processors/hooks/content/content-hooks-frames-web.js"],
+	output: [{
+		file: "lib/single-file-hooks-frames.js",
+		format: "iife",
+		plugins: []
+	}],
+	plugins: PLUGINS,
+	external: EXTERNAL
+}, {
+	input: ["single-file-core/common/content-infobar-web.js"],
+	output: [{
+		file: "lib/single-file-infobar.js",
+		format: "iife",
+		plugins: [terser()]
+	}],
+	plugins: PLUGINS,
+	external: EXTERNAL
+}];

package/rollup.config.js

@@ -0,0 +1,64 @@
+import { terser } from "rollup-plugin-terser";
+import resolve from "@rollup/plugin-node-resolve";
+
+const PLUGINS = [resolve({ moduleDirectories: ["node_modules"] })];
+const EXTERNAL = ["single-file-core"];
+
+export default [{
+	input: ["single-file-core/index.js"],
+	output: [{
+		file: "lib/single-file.js",
+		format: "umd",
+		name: "singlefile",
+		plugins: [terser()]
+	}],
+	plugins: PLUGINS,
+	external: EXTERNAL
+}, {
+	input: ["single-file-core/processors/frame-tree/content/content-frame-tree.js"],
+	output: [{
+		file: "lib/single-file-frames.js",
+		format: "umd",
+		name: "singlefile",
+		plugins: [terser()]
+	}],
+	plugins: PLUGINS,
+	external: EXTERNAL
+}, {
+	input: ["single-file-core/single-file-bootstrap.js"],
+	output: [{
+		file: "lib/single-file-bootstrap.js",
+		format: "umd",
+		name: "singlefileBootstrap",
+		plugins: [terser()]
+	}],
+	plugins: PLUGINS,
+	external: EXTERNAL
+}, {
+	input: ["single-file-core/processors/hooks/content/content-hooks-web.js"],
+	output: [{
+		file: "lib/single-file-hooks.js",
+		format: "iife",
+		plugins: [terser()]
+	}],
+	plugins: PLUGINS,
+	external: EXTERNAL
+}, {
+	input: ["single-file-core/processors/hooks/content/content-hooks-frames-web.js"],
+	output: [{
+		file: "lib/single-file-hooks-frames.js",
+		format: "iife",
+		plugins: [terser()]
+	}],
+	plugins: PLUGINS,
+	external: EXTERNAL
+}, {
+	input: ["single-file-core/common/content-infobar-web.js"],
+	output: [{
+		file: "lib/single-file-infobar.js",
+		format: "iife",
+		plugins: [terser()]
+	}],
+	plugins: PLUGINS,
+	external: EXTERNAL
+}];

package/single-file

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+
+/*
+ * Copyright 2010-2020 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ *
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or
+ *   modify it under the terms of the GNU Affero General Public License
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ *
+ *   The code in this file is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU
+ *   AGPL normally required by section 4, provided you include this license
+ *   notice and a URL through which recipients can access the Corresponding
+ *   Source.
+ */
+
+/* global require */
+
+const fileUrl = require("file-url");
+const fs = require("fs");
+const api = require("./single-file-cli-api");
+run(require("./args"))
+	.catch(error => console.error(error.message || error)); // eslint-disable-line no-console	
+
+async function run(options) {
+	let urls;
+	if (options.url && !api.VALID_URL_TEST.test(options.url)) {
+		options.url = fileUrl(options.url);
+	}
+	if (options.urlsFile) {
+		urls = fs.readFileSync(options.urlsFile).toString().split("\n");
+	} else {
+		urls = [options.url];
+	}
+	if (options.browserCookiesFile) {
+		const cookiesContent = fs.readFileSync(options.browserCookiesFile).toString();
+		try {
+			options.browserCookies = JSON.parse(cookiesContent);
+		} catch (error) {
+			options.browserCookies = parseCookies(cookiesContent);
+		}
+	}
+	options.retrieveLinks = true;
+	const singlefile = await api.initialize(options);
+	await singlefile.capture(urls);
+	await singlefile.finish();
+}
+
+function parseCookies(textValue) {
+	const httpOnlyRegExp = /^#HttpOnly_(.*)/;
+	return textValue.split(/\r\n|\n/)
+		.filter(line => line.trim() && (!/^#/.test(line) || httpOnlyRegExp.test(line)))
+		.map(line => {
+			const httpOnly = httpOnlyRegExp.test(line);
+			if (httpOnly) {
+				line = line.replace(httpOnlyRegExp, "$1");
+			}
+			const values = line.split(/\t/);
+			if (values.length == 7) {
+				return {
+					domain: values[0],
+					path: values[2],
+					secure: values[3] == "TRUE",
+					expires: (values[4] && Number(values[4])) || undefined,
+					name: values[5],
+					value: values[6],
+					httpOnly
+				};
+			}
+		})
+		.filter(cookieData => cookieData);
+}

package/single-file-cli-api.js

@@ -0,0 +1,324 @@
+/*
+ * Copyright 2010-2020 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ *
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or
+ *   modify it under the terms of the GNU Affero General Public License
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ *
+ *   The code in this file is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU
+ *   AGPL normally required by section 4, provided you include this license
+ *   notice and a URL through which recipients can access the Corresponding
+ *   Source.
+ */
+
+/* global require, exports, URL */
+
+const fs = require("fs");
+const path = require("path");
+const scripts = require("./back-ends/common/scripts.js");
+const VALID_URL_TEST = /^(https?|file):\/\//;
+
+const DEFAULT_OPTIONS = {
+	removeHiddenElements: true,
+	removeUnusedStyles: true,
+	removeUnusedFonts: true,
+	removeFrames: false,
+	removeImports: true,
+	compressHTML: true,
+	compressCSS: false,
+	loadDeferredImages: true,
+	loadDeferredImagesMaxIdleTime: 1500,
+	loadDeferredImagesBlockCookies: false,
+	loadDeferredImagesBlockStorage: false,
+	loadDeferredImagesKeepZoomLevel: false,
+	loadDeferredImagesDispatchScrollEvent: false,
+	filenameTemplate: "{page-title} ({date-locale} {time-locale}).html",
+	infobarTemplate: "",
+	includeInfobar: false,
+	filenameMaxLength: 192,
+	filenameMaxLengthUnit: "bytes",
+	filenameReplacedCharacters: ["~", "+", "\\\\", "?", "%", "*", ":", "|", "\"", "<", ">", "\x00-\x1f", "\x7F"],
+	filenameReplacementCharacter: "_",
+	maxResourceSizeEnabled: false,
+	maxResourceSize: 10,
+	backgroundSave: true,
+	removeAlternativeFonts: true,
+	removeAlternativeMedias: true,
+	removeAlternativeImages: true,
+	groupDuplicateImages: true,
+	saveRawPage: false,
+	resolveFragmentIdentifierURLs: false,
+	userScriptEnabled: false,
+	saveFavicon: true,
+	includeBOM: false,
+	insertMetaCSP: true,
+	insertMetaNoIndex: false,
+	insertSingleFileComment: true,
+	blockImages: false,
+	blockStylesheets: false,
+	blockFont: false,
+	blockScripts: true,
+	blockVideos: true,
+	blockAudios: true
+};
+const STATE_PROCESSING = "processing";
+const STATE_PROCESSED = "processed";
+
+const backEnds = {
+	jsdom: "./back-ends/jsdom.js",
+	puppeteer: "./back-ends/puppeteer.js",
+	"puppeteer-firefox": "./back-ends/puppeteer-firefox.js",
+	"webdriver-chromium": "./back-ends/webdriver-chromium.js",
+	"webdriver-gecko": "./back-ends/webdriver-gecko.js",
+	"playwright-firefox": "./back-ends/playwright-firefox.js",
+	"playwright-chromium": "./back-ends/playwright-chromium.js"
+};
+
+let backend, tasks = [], maxParallelWorkers = 8, sessionFilename;
+
+exports.getBackEnd = backEndName => require(backEnds[backEndName]);
+exports.DEFAULT_OPTIONS = DEFAULT_OPTIONS;
+exports.VALID_URL_TEST = VALID_URL_TEST;
+exports.initialize = initialize;
+
+async function initialize(options) {
+	options = Object.assign({}, DEFAULT_OPTIONS, options);
+	maxParallelWorkers = options.maxParallelWorkers;
+	backend = require(backEnds[options.backEnd]);
+	await backend.initialize(options);
+	if (options.crawlSyncSession || options.crawlLoadSession) {
+		try {
+			tasks = JSON.parse(fs.readFileSync(options.crawlSyncSession || options.crawlLoadSession).toString());
+		} catch (error) {
+			if (options.crawlLoadSession) {
+				throw error;
+			}
+		}
+	}
+	if (options.crawlSyncSession || options.crawlSaveSession) {
+		sessionFilename = options.crawlSyncSession || options.crawlSaveSession;
+	}
+	return {
+		capture: urls => capture(urls, options),
+		finish: () => finish(options),
+	};
+}
+
+async function capture(urls, options) {
+	let newTasks;
+	const taskUrls = tasks.map(task => task.url);
+	newTasks = urls.map(url => createTask(url, options));
+	newTasks = newTasks.filter(task => task && !taskUrls.includes(task.url));
+	if (newTasks.length) {
+		tasks = tasks.concat(newTasks);
+		saveTasks();
+	}
+	await runTasks();
+}
+
+async function finish(options) {
+	const promiseTasks = tasks.map(task => task.promise);
+	await Promise.all(promiseTasks);
+	if (options.crawlReplaceURLs) {
+		tasks.forEach(task => {
+			try {
+				let pageContent = fs.readFileSync(task.filename).toString();
+				tasks.forEach(otherTask => {
+					if (otherTask.filename) {
+						pageContent = pageContent.replace(new RegExp(escapeRegExp("\"" + otherTask.originalUrl + "\""), "gi"), "\"" + otherTask.filename + "\"");
+						pageContent = pageContent.replace(new RegExp(escapeRegExp("'" + otherTask.originalUrl + "'"), "gi"), "'" + otherTask.filename + "'");
+						const filename = otherTask.filename.replace(/ /g, "%20");
+						pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + otherTask.originalUrl + " "), "gi"), "=" + filename + " ");
+						pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + otherTask.originalUrl + ">"), "gi"), "=" + filename + ">");
+					}
+				});
+				fs.writeFileSync(task.filename, pageContent);
+			} catch (error) {
+				// ignored
+			}
+		});
+	}
+	if (!options.browserDebug) {
+		return backend.closeBrowser();
+	}
+}
+
+async function runTasks() {
+	const availableTasks = tasks.filter(task => !task.status).length;
+	const processingTasks = tasks.filter(task => task.status == STATE_PROCESSING).length;
+	const promisesTasks = [];
+	for (let workerIndex = 0; workerIndex < Math.min(availableTasks, maxParallelWorkers - processingTasks); workerIndex++) {
+		promisesTasks.push(runNextTask());
+	}
+	return Promise.all(promisesTasks);
+}
+
+async function runNextTask() {
+	const task = tasks.find(task => !task.status);
+	if (task) {
+		const options = task.options;
+		let taskOptions = JSON.parse(JSON.stringify(options));
+		taskOptions.url = task.url;
+		task.status = STATE_PROCESSING;
+		saveTasks();
+		task.promise = capturePage(taskOptions);
+		const pageData = await task.promise;
+		task.status = STATE_PROCESSED;
+		if (pageData) {
+			task.filename = pageData.filename;
+			if (options.crawlLinks && testMaxDepth(task)) {
+				let newTasks = pageData.links
+					.map(urlLink => createTask(urlLink, options, task, tasks[0]))
+					.filter(task => task &&
+						testMaxDepth(task) &&
+						!tasks.find(otherTask => otherTask.url == task.url) &&
+						(!options.crawlInnerLinksOnly || task.isInnerLink) &&
+						(!options.crawlNoParent || (task.isChild || !task.isInnerLink)));
+				tasks.splice(tasks.length, 0, ...newTasks);
+			}
+		}
+		saveTasks();
+		await runTasks();
+	}
+}
+
+function testMaxDepth(task) {
+	const options = task.options;
+	return (options.crawlMaxDepth == 0 || task.depth <= options.crawlMaxDepth) &&
+		(options.crawlExternalLinksMaxDepth == 0 || task.externalLinkDepth < options.crawlExternalLinksMaxDepth);
+}
+
+function createTask(url, options, parentTask, rootTask) {
+	url = parentTask ? rewriteURL(url, options.crawlRemoveURLFragment, options.crawlRewriteRules) : url;
+	if (VALID_URL_TEST.test(url)) {
+		const isInnerLink = rootTask && url.startsWith(getHostURL(rootTask.url));
+		const rootBaseURIMatch = rootTask && rootTask.url.match(/(.*?)[^/]*$/);
+		const isChild = isInnerLink && rootBaseURIMatch && rootBaseURIMatch[1] && url.startsWith(rootBaseURIMatch[1]);
+		return {
+			url,
+			isInnerLink,
+			isChild,
+			originalUrl: url,
+			rootBaseURI: rootBaseURIMatch && rootBaseURIMatch[1],
+			depth: parentTask ? parentTask.depth + 1 : 0,
+			externalLinkDepth: isInnerLink ? -1 : parentTask ? parentTask.externalLinkDepth + 1 : -1,
+			options
+		};
+	}
+}
+
+function saveTasks() {
+	if (sessionFilename) {
+		fs.writeFileSync(sessionFilename, JSON.stringify(
+			tasks.map(task => Object.assign({}, task, {
+				status: task.status == STATE_PROCESSING ? undefined : task.status,
+				promise: undefined,
+				options: task.status && task.status == STATE_PROCESSED ? undefined : task.options
+			}))
+		));
+	}
+}
+
+function rewriteURL(url, crawlRemoveURLFragment, crawlRewriteRules) {
+	url = url.trim();
+	if (crawlRemoveURLFragment) {
+		url = url.replace(/^(.*?)#.*$/, "$1");
+	}
+	crawlRewriteRules.forEach(rewriteRule => {
+		const parts = rewriteRule.trim().split(/ +/);
+		if (parts.length) {
+			url = url.replace(new RegExp(parts[0]), parts[1] || "").trim();
+		}
+	});
+	return url;
+}
+
+function getHostURL(url) {
+	url = new URL(url);
+	return url.protocol + "//" + (url.username ? url.username + (url.password || "") + "@" : "") + url.hostname;
+}
+
+async function capturePage(options) {
+	try {
+		let filename;
+		const pageData = await backend.getPageData(options);
+		if (options.includeInfobar) {
+			await includeInfobarScript(pageData);
+		}
+		if (options.output) {
+			filename = getFilename(options.output, options);
+		} else if (options.dumpContent) {
+			console.log(pageData.content); // eslint-disable-line no-console
+		} else {
+			filename = getFilename(pageData.filename, options);
+		}
+		if (filename) {
+			const dirname = path.dirname(filename);
+			if (dirname) {
+				fs.mkdirSync(dirname, { recursive: true });
+			}
+			fs.writeFileSync(filename, pageData.content);
+		}
+		return pageData;
+	} catch (error) {
+		const message = "URL: " + options.url + "\nStack: " + error.stack + "\n";
+		if (options.errorFile) {
+			fs.writeFileSync(options.errorFile, message, { flag: "a" });
+		} else {
+			console.error(error.message || error, message); // eslint-disable-line no-console
+		}
+	}
+}
+
+function getFilename(filename, options, index = 1) {
+	if (Array.isArray(options.outputDirectory)) {
+		const outputDirectory = options.outputDirectory.pop();
+		if (outputDirectory.startsWith("/")) {
+			options.outputDirectory = outputDirectory;
+		} else {
+			options.outputDirectory = options.outputDirectory[0] + outputDirectory;
+		}
+	}
+	let outputDirectory = options.outputDirectory || "";
+	if (outputDirectory && !outputDirectory.endsWith("/")) {
+		outputDirectory += "/";
+	}
+	let newFilename = outputDirectory + filename;
+	if (options.filenameConflictAction == "overwrite") {
+		return filename;
+	} else if (options.filenameConflictAction == "uniquify" && index > 1) {
+		const regExpMatchExtension = /(\.[^.]+)$/;
+		const matchExtension = newFilename.match(regExpMatchExtension);
+		if (matchExtension && matchExtension[1]) {
+			newFilename = newFilename.replace(regExpMatchExtension, " (" + index + ")" + matchExtension[1]);
+		} else {
+			newFilename += " (" + index + ")";
+		}
+	}
+	if (fs.existsSync(newFilename)) {
+		if (options.filenameConflictAction != "skip") {
+			return getFilename(filename, options, index + 1);
+		}
+	} else {
+		return newFilename;
+	}
+}
+
+function escapeRegExp(string) {
+	return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+async function includeInfobarScript(pageData) {
+	const infobarContent = await scripts.getInfobarScript();
+	pageData.content += "<script>document.currentScript.remove();" + infobarContent + "</script>";
+}

package/single-file.bat

@@ -0,0 +1,2 @@
+@echo off
+node "%~dp0\single-file" %*