single-file-cli 1.1.53 → 2.0.2

package/package.json

@@ -1,41 +1,17 @@
 {
 	"name": "single-file-cli",
-	"version": "1.1.53",
+	"version": "2.0.2",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
-	"license": "AGPL-3.0-or-later",
-	"main": "single-file-cli-api.js",
-	"bin": {
-		"single-file": "./single-file"
+	"engines": {
+		"deno": ">=1.4",
+		"bun": ">=1.0",
+		"node": ">=20"
 	},
-	"scripts": {
-		"build": "./build-lib.sh",
-		"prepublishOnly": "./build-lib.sh"
-	},
-	"repository": {
-		"type": "git",
-		"url": "git+https://github.com/gildas-lormeau/single-file-cli.git"
-	},
-	"bugs": {
-		"url": "https://github.com/gildas-lormeau/single-file-cli/issues"
-	},
-	"homepage": "https://github.com/gildas-lormeau/single-file-cli#readme",
+	"type": "module",
+	"bin": "./single-file",
 	"dependencies": {
-		"file-url": "3.0.0",
-		"iconv-lite": "0.6.3",
-		"jsdom": "24.0.0",
-		"puppeteer-core": "22.0.0",
-		"selenium-webdriver": "4.17.0",
-		"single-file-core": "1.3.23",
-		"strong-data-uri": "1.0.6",
-		"yargs": "17.7.2"
-	},
-	"devDependencies": {
-		"@rollup/plugin-node-resolve": "15.0.1",
-		"rollup": "2.79.1",
-		"rollup-plugin-terser": "7.0.2"
-	},
-	"overrides": {
-		"terser": "^5.15.0"
+		"simple-cdp": "^1.8.2",
+		"ws": "^8.16.0"
 	}
 }

package/single-file

@@ -1,7 +1,7 @@
-#!/usr/bin/env node
+#!/usr/bin/env -S deno run --allow-read --allow-write --allow-net --allow-env --allow-run
 
 /*
- * Copyright 2010-2020 Gildas Lormeau
+ * Copyright 2010-2024 Gildas Lormeau
  * contact : gildas.lormeau <at> gmail.com
  *
  * This file is part of SingleFile.
@@ -23,34 +23,59 @@
  *   Source.
  */
 
-/* global require */
+import { VALID_URL_TEST, initialize } from "./single-file-cli-api.js";
+import { initGlobalThisProperties, readTextFile, toFileUrl } from "./deno-polyfill.js";
+import args from "./args.js";
 
-const fileUrl = require("file-url");
-const fs = require("fs");
-const api = require("./single-file-cli-api");
-run(require("./args"))
-	.catch(error => console.error(error.message || error)); // eslint-disable-line no-console	
+await initGlobalThisProperties();
+await run(args);
 
-async function run(options) {
+async function run(options = {}) {
 	let urls;
-	if (options.url && !api.VALID_URL_TEST.test(options.url)) {
-		options.url = fileUrl(options.url);
+	if (options.url && !VALID_URL_TEST.test(options.url)) {
+		options.url = (await toFileUrl(new URL(options.url, import.meta.url).pathname)).href;
 	}
 	if (options.urlsFile) {
-		urls = fs.readFileSync(options.urlsFile).toString().split("\n");
+		urls = (await readTextFile(options.urlsFile)).split("\n");
 	} else {
 		urls = [options.url];
 	}
+	if (options.browserCookies) {
+		const cookies = [];
+		for (const cookie of options.browserCookies) {
+			const [name, value, domain, path, expires, httpOnly, secure, sameSite, url] = cookie.split(",");
+			cookies.push({
+				name,
+				value,
+				url,
+				domain,
+				path,
+				secure,
+				httpOnly,
+				sameSite,
+				expires
+			});
+		}
+		options.browserCookies = cookies;
+	}
 	if (options.browserCookiesFile) {
-		const cookiesContent = fs.readFileSync(options.browserCookiesFile).toString();
+		const cookiesContent = await readTextFile(options.browserCookiesFile);
 		try {
 			options.browserCookies = JSON.parse(cookiesContent);
 		} catch (error) {
 			options.browserCookies = parseCookies(cookiesContent);
 		}
 	}
+	if (options.httpHeaders) {
+		const headers = {};
+		for (const header of options.httpHeaders) {
+			const [name, value] = header.split("=");
+			headers[name] = value.trim();
+		}
+		options.httpHeaders = headers;
+	}
 	options.retrieveLinks = true;
-	const singlefile = await api.initialize(options);
+	const singlefile = await initialize(options);
 	await singlefile.capture(urls);
 	await singlefile.finish();
 }

package/single-file-cli-api.js

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2020 Gildas Lormeau
+ * Copyright 2010-2024 Gildas Lormeau
  * contact : gildas.lormeau <at> gmail.com
  *
  * This file is part of SingleFile.
@@ -21,10 +21,12 @@
  *   Source.
  */
 
-/* global require, exports, URL, process */
+/* global URL */
+
+import * as backend from "./back-ends/chromium-back-end.js";
+import { getZipScriptSource } from "./back-ends/single-file-script.js";
+import { readTextFile, writeTextFile, stdout, mkdir, stat, dirname } from "./deno-polyfill.js";
 
-const fs = require("fs");
-const path = require("path");
 const VALID_URL_TEST = /^(https?|file):\/\//;
 
 const DEFAULT_OPTIONS = {
@@ -74,32 +76,17 @@ const DEFAULT_OPTIONS = {
 const STATE_PROCESSING = "processing";
 const STATE_PROCESSED = "processed";
 
-const backEnds = {
-	jsdom: "./back-ends/jsdom.js",
-	puppeteer: "./back-ends/puppeteer.js",
-	"puppeteer-firefox": "./back-ends/puppeteer-firefox.js",
-	"webdriver-chromium": "./back-ends/webdriver-chromium.js",
-	"webdriver-gecko": "./back-ends/webdriver-gecko.js",
-	"playwright-firefox": "./back-ends/playwright-firefox.js",
-	"playwright-chromium": "./back-ends/playwright-chromium.js",
-	"playwright-webkit": "./back-ends/playwright-webkit.js"
-};
-
-let backend, tasks = [], maxParallelWorkers = 8, sessionFilename;
+let tasks = [], maxParallelWorkers, sessionFilename;
 
-exports.getBackEnd = backEndName => require(backEnds[backEndName]);
-exports.DEFAULT_OPTIONS = DEFAULT_OPTIONS;
-exports.VALID_URL_TEST = VALID_URL_TEST;
-exports.initialize = initialize;
+export { DEFAULT_OPTIONS, VALID_URL_TEST, initialize };
 
 async function initialize(options) {
 	options = Object.assign({}, DEFAULT_OPTIONS, options);
-	maxParallelWorkers = options.maxParallelWorkers;
-	backend = require(backEnds[options.backEnd]);
+	maxParallelWorkers = options.maxParallelWorkers || 8;
 	await backend.initialize(options);
 	if (options.crawlSyncSession || options.crawlLoadSession) {
 		try {
-			tasks = JSON.parse(fs.readFileSync(options.crawlSyncSession || options.crawlLoadSession).toString());
+			tasks = JSON.parse(await readTextFile(options.crawlSyncSession || options.crawlLoadSession));
 		} catch (error) {
 			if (options.crawlLoadSession) {
 				throw error;
@@ -122,7 +109,7 @@ async function capture(urls, options) {
 	newTasks = newTasks.filter(task => task && !taskUrls.includes(task.url));
 	if (newTasks.length) {
 		tasks = tasks.concat(newTasks);
-		saveTasks();
+		await saveTasks();
 	}
 	await runTasks();
 }
@@ -131,9 +118,9 @@ async function finish(options) {
 	const promiseTasks = tasks.map(task => task.promise);
 	await Promise.all(promiseTasks);
 	if (options.crawlReplaceURLs && !options.compressContent) {
-		tasks.forEach(task => {
+		for (const task of tasks) {
 			try {
-				let pageContent = fs.readFileSync(task.filename).toString();
+				let pageContent = await readTextFile(task.filename);
 				tasks.forEach(otherTask => {
 					if (otherTask.filename) {
 						pageContent = pageContent.replace(new RegExp(escapeRegExp("\"" + otherTask.originalUrl + "\""), "gi"), "\"" + otherTask.filename + "\"");
@@ -143,11 +130,11 @@ async function finish(options) {
 						pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + otherTask.originalUrl + ">"), "gi"), "=" + filename + ">");
 					}
 				});
-				fs.writeFileSync(task.filename, pageContent);
+				await writeTextFile(task.filename, pageContent);
 			} catch (error) {
 				// ignored
 			}
-		});
+		}
 	}
 	if (!options.browserDebug) {
 		return backend.closeBrowser();
@@ -171,7 +158,7 @@ async function runNextTask() {
 		let taskOptions = JSON.parse(JSON.stringify(options));
 		taskOptions.url = task.url;
 		task.status = STATE_PROCESSING;
-		saveTasks();
+		await saveTasks();
 		task.promise = capturePage(taskOptions);
 		const pageData = await task.promise;
 		task.status = STATE_PROCESSED;
@@ -188,7 +175,7 @@ async function runNextTask() {
 				tasks.splice(tasks.length, 0, ...newTasks);
 			}
 		}
-		saveTasks();
+		await saveTasks();
 		await runTasks();
 	}
 }
@@ -218,9 +205,9 @@ function createTask(url, options, parentTask, rootTask) {
 	}
 }
 
-function saveTasks() {
+async function saveTasks() {
 	if (sessionFilename) {
-		fs.writeFileSync(sessionFilename, JSON.stringify(
+		await writeTextFile(sessionFilename, JSON.stringify(
 			tasks.map(task => Object.assign({}, task, {
 				status: task.status == STATE_PROCESSING ? undefined : task.status,
 				promise: undefined,
@@ -252,38 +239,38 @@ function getHostURL(url) {
 async function capturePage(options) {
 	try {
 		let filename;
-		options.zipScript = fs.readFileSync(require.resolve("./lib/single-file-zip.min.js")).toString();
+		options.zipScript = getZipScriptSource();
 		const pageData = await backend.getPageData(options);
 		if (options.output) {
-			filename = getFilename(options.output, options);
+			filename = await getFilename(options.output, options);
 		} else if (options.dumpContent) {
 			if (options.compressContent) {
-				await new Promise(resolve => process.stdout.write(pageData.content, resolve));
+				await stdout.write(pageData.content);
 			} else {
 				console.log(pageData.content); // eslint-disable-line no-console
 			}
 		} else {
-			filename = getFilename(pageData.filename, options);
+			filename = await getFilename(pageData.filename, options);
 		}
 		if (filename) {
-			const dirname = path.dirname(filename);
-			if (dirname) {
-				fs.mkdirSync(dirname, { recursive: true });
+			const directoryName = await dirname(filename);
+			if (directoryName !== ".") {
+				await mkdir(directoryName, { recursive: true });
 			}
-			fs.writeFileSync(filename, pageData.content);
+			await writeTextFile(filename, pageData.content);
 		}
 		return pageData;
 	} catch (error) {
 		const message = "URL: " + options.url + "\nStack: " + error.stack + "\n";
 		if (options.errorFile) {
-			fs.writeFileSync(options.errorFile, message, { flag: "a" });
+			await writeTextFile(options.errorFile, message, { append: true });
 		} else {
 			console.error(error.message || error, message); // eslint-disable-line no-console
 		}
 	}
 }
 
-function getFilename(filename, options, index = 1) {
+async function getFilename(filename, options, index = 1) {
 	if (Array.isArray(options.outputDirectory)) {
 		const outputDirectory = options.outputDirectory.pop();
 		if (outputDirectory.startsWith("/")) {
@@ -308,11 +295,12 @@ function getFilename(filename, options, index = 1) {
 			newFilename += " (" + index + ")";
 		}
 	}
-	if (fs.existsSync(newFilename)) {
+	try {
+		await stat(newFilename);
 		if (options.filenameConflictAction != "skip") {
 			return getFilename(filename, options, index + 1);
 		}
-	} else {
+	} catch (_) {
 		return newFilename;
 	}
 }

package/back-ends/extensions/bypass-csp/index.js

@@ -1,36 +0,0 @@
-/*
- * Copyright 2010-2020 Gildas Lormeau
- * contact : gildas.lormeau <at> gmail.com
- * 
- * This file is part of SingleFile.
- *
- *   The code in this file is free software: you can redistribute it and/or 
- *   modify it under the terms of the GNU Affero General Public License 
- *   (GNU AGPL) as published by the Free Software Foundation, either version 3
- *   of the License, or (at your option) any later version.
- * 
- *   The code in this file is distributed in the hope that it will be useful, 
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
- *   General Public License for more details.
- *
- *   As additional permission under GNU AGPL version 3 section 7, you may 
- *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
- *   AGPL normally required by section 4, provided you include this license 
- *   notice and a URL through which recipients can access the Corresponding 
- *   Source.
- */
-
-const removedHeaders = ["content-security-policy"];
-
-const browserAPI = this.browser || this.chrome;
-
-browserAPI.webRequest.onHeadersReceived.addListener(
-	function (details) {
-		let responseHeaders = details.responseHeaders;
-		responseHeaders = responseHeaders.filter(responseHeader => !removedHeaders.includes(responseHeader.name.toLowerCase()));
-		return { responseHeaders };
-	},
-	{ urls: ["<all_urls>"] },
-	["blocking", "responseHeaders"]
-);

package/back-ends/extensions/bypass-csp/manifest.json

@@ -1,20 +0,0 @@
-{
-	"name": "bypass-csp",
-	"version": "0.0.3",
-	"background": {
-		"scripts": [
-			"index.js"
-		]
-	},
-	"permissions": [
-		"webRequest",
-		"webRequestBlocking",
-		"<all_urls>"
-	],
-	"browser_specific_settings": {
-		"gecko": {
-			"id": "{55e2789c-817b-4d75-815b-df6921c84ed8}"
-		}
-	},
-	"manifest_version": 2
-}

package/back-ends/extensions/disable-web-security/index.js

@@ -1,54 +0,0 @@
-/*
- * Copyright 2010-2020 Gildas Lormeau
- * contact : gildas.lormeau <at> gmail.com
- * 
- * This file is part of SingleFile.
- *
- *   The code in this file is free software: you can redistribute it and/or 
- *   modify it under the terms of the GNU Affero General Public License 
- *   (GNU AGPL) as published by the Free Software Foundation, either version 3
- *   of the License, or (at your option) any later version.
- * 
- *   The code in this file is distributed in the hope that it will be useful, 
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
- *   General Public License for more details.
- *
- *   As additional permission under GNU AGPL version 3 section 7, you may 
- *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
- *   AGPL normally required by section 4, provided you include this license 
- *   notice and a URL through which recipients can access the Corresponding 
- *   Source.
- */
-
-const removedHeaders = ["access-control-allow-methods", "access-control-allow-headers"];
-const updatedHeaders = { "access-control-allow-origin": "*", "access-control-allow-credentials": "true" };
-
-const browserAPI = this.browser || this.chrome;
-
-browserAPI.webRequest.onHeadersReceived.addListener(
-	function (details) {
-		let responseHeaders = details.responseHeaders;
-		let processedHeaders = [];
-		responseHeaders = responseHeaders.filter(responseHeader => !removedHeaders.includes(responseHeader.name.toLowerCase()));
-		responseHeaders.forEach(responseHeader => {
-			const name = responseHeader.name.toLowerCase();
-			const value = updatedHeaders[name];
-			if (value) {
-				responseHeader.value = value;
-				processedHeaders.push(name);
-			}
-		});
-		Object.keys(updatedHeaders).forEach(name => {
-			if (!processedHeaders.includes(name)) {
-				const value = updatedHeaders[name.toLowerCase()];
-				if (value) {
-					responseHeaders.push({ name, value });
-				}
-			}
-		});
-		return { responseHeaders };
-	},
-	{ urls: ["<all_urls>"] },
-	["blocking", "responseHeaders"]
-);

package/back-ends/extensions/disable-web-security/manifest.json

@@ -1,20 +0,0 @@
-{
-	"name": "disable-web-security",
-	"version": "0.0.3",
-	"background": {
-		"scripts": [
-			"index.js"
-		]
-	},
-	"permissions": [
-		"webRequest",
-		"webRequestBlocking",
-		"<all_urls>"
-	],
-	"browser_specific_settings": {
-		"gecko": {
-			"id": "{588434c2-67c3-4f77-9828-c30c7d63e8f9}"
-		}
-	},
-	"manifest_version": 2
-}

package/back-ends/extensions/network-idle/bg.js

@@ -1,61 +0,0 @@
-/*
- * Copyright 2010-2020 Gildas Lormeau
- * contact : gildas.lormeau <at> gmail.com
- * 
- * This file is part of SingleFile.
- *
- *   The code in this file is free software: you can redistribute it and/or 
- *   modify it under the terms of the GNU Affero General Public License 
- *   (GNU AGPL) as published by the Free Software Foundation, either version 3
- *   of the License, or (at your option) any later version.
- * 
- *   The code in this file is distributed in the hope that it will be useful, 
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
- *   General Public License for more details.
- *
- *   As additional permission under GNU AGPL version 3 section 7, you may 
- *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
- *   AGPL normally required by section 4, provided you include this license 
- *   notice and a URL through which recipients can access the Corresponding 
- *   Source.
- */
-
-/* global setTimeout, clearTimeout */
-
-const browserAPI = this.browser || this.chrome;
-
-const IDLE_DELAY = 1000;
-const watchDogs = [];
-let pendingRequests = new Set();
-
-browserAPI.webRequest.onSendHeaders.addListener(onRequest, { urls: ["<all_urls>"] }, []);
-browserAPI.webRequest.onResponseStarted.addListener(onResponse, { urls: ["<all_urls>"] }, []);
-browserAPI.webRequest.onErrorOccurred.addListener(onResponse, { urls: ["<all_urls>"] });
-
-function onRequest(details) {
-	if (details.tabId != -1) {
-		pendingRequests.add(details.requestId);
-		if (pendingRequests.size > 2) {
-			clearTimeout(watchDogs[2]);
-		}
-		clearTimeout(watchDogs[0]);
-	}
-}
-
-function onResponse(details) {
-	if (details.tabId != -1) {
-		pendingRequests.delete(details.requestId);
-		if (pendingRequests.size == 2) {
-			maybeIdle(2, details.tabId);
-		}
-		if (pendingRequests.size == 0) {
-			maybeIdle(0, details.tabId);
-		}
-	}
-}
-
-function maybeIdle(idleLevel, tabId) {
-	clearTimeout(watchDogs[idleLevel]);
-	watchDogs[idleLevel] = setTimeout(() => browserAPI.tabs.sendMessage(tabId, "network-idle-" + idleLevel), IDLE_DELAY);
-}

package/back-ends/extensions/network-idle/content.js

@@ -1,29 +0,0 @@
-/*
- * Copyright 2010-2020 Gildas Lormeau
- * contact : gildas.lormeau <at> gmail.com
- * 
- * This file is part of SingleFile.
- *
- *   The code in this file is free software: you can redistribute it and/or 
- *   modify it under the terms of the GNU Affero General Public License 
- *   (GNU AGPL) as published by the Free Software Foundation, either version 3
- *   of the License, or (at your option) any later version.
- * 
- *   The code in this file is distributed in the hope that it will be useful, 
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
- *   General Public License for more details.
- *
- *   As additional permission under GNU AGPL version 3 section 7, you may 
- *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
- *   AGPL normally required by section 4, provided you include this license 
- *   notice and a URL through which recipients can access the Corresponding 
- *   Source.
- */
-
-/* global window, CustomEvent */
-
-const browserAPI = this.browser || this.chrome;
-const dispatchEvent = event => { try { window.dispatchEvent(event); } catch (error) {  /* ignored */ } };
-
-browserAPI.runtime.onMessage.addListener(message => dispatchEvent(new CustomEvent("single-file-" + message)));

package/back-ends/extensions/network-idle/manifest.json

@@ -1,31 +0,0 @@
-{
-	"name": "network-idle",
-	"version": "0.0.2",
-	"background": {
-		"scripts": [
-			"bg.js"
-		]
-	},
-	"content_scripts": [
-		{
-			"matches": [
-				"<all_urls>"
-			],
-			"run_at": "document_start",
-			"js": [
-				"content.js"
-			]
-		}
-	],
-	"permissions": [
-		"tabs",
-		"webRequest",
-		"<all_urls>"
-	],
-	"browser_specific_settings": {
-		"gecko": {
-			"id": "{caf9306a-8951-4f0c-beb0-bab690d00caf}"
-		}
-	},
-	"manifest_version": 2
-}