single-file-cli 1.0.0

package/back-ends/extensions/disable-web-security/index.js

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2010-2020 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ * 
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or 
+ *   modify it under the terms of the GNU Affero General Public License 
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ * 
+ *   The code in this file is distributed in the hope that it will be useful, 
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may 
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
+ *   AGPL normally required by section 4, provided you include this license 
+ *   notice and a URL through which recipients can access the Corresponding 
+ *   Source.
+ */
+
+const removedHeaders = ["access-control-allow-methods", "access-control-allow-headers"];
+const updatedHeaders = { "access-control-allow-origin": "*", "access-control-allow-credentials": "true" };
+
+const browserAPI = this.browser || this.chrome;
+
+browserAPI.webRequest.onHeadersReceived.addListener(
+	function (details) {
+		let responseHeaders = details.responseHeaders;
+		let processedHeaders = [];
+		responseHeaders = responseHeaders.filter(responseHeader => !removedHeaders.includes(responseHeader.name.toLowerCase()));
+		responseHeaders.forEach(responseHeader => {
+			const name = responseHeader.name.toLowerCase();
+			const value = updatedHeaders[name];
+			if (value) {
+				responseHeader.value = value;
+				processedHeaders.push(name);
+			}
+		});
+		Object.keys(updatedHeaders).forEach(name => {
+			if (!processedHeaders.includes(name)) {
+				const value = updatedHeaders[name.toLowerCase()];
+				if (value) {
+					responseHeaders.push({ name, value });
+				}
+			}
+		});
+		return { responseHeaders };
+	},
+	{ urls: ["<all_urls>"] },
+	["blocking", "responseHeaders"]
+);

package/back-ends/extensions/disable-web-security/manifest.json

@@ -0,0 +1,20 @@
+{
+	"name": "disable-web-security",
+	"version": "0.0.3",
+	"background": {
+		"scripts": [
+			"index.js"
+		]
+	},
+	"permissions": [
+		"webRequest",
+		"webRequestBlocking",
+		"<all_urls>"
+	],
+	"browser_specific_settings": {
+		"gecko": {
+			"id": "{588434c2-67c3-4f77-9828-c30c7d63e8f9}"
+		}
+	},
+	"manifest_version": 2
+}

package/back-ends/extensions/network-idle/bg.js

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2010-2020 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ * 
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or 
+ *   modify it under the terms of the GNU Affero General Public License 
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ * 
+ *   The code in this file is distributed in the hope that it will be useful, 
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may 
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
+ *   AGPL normally required by section 4, provided you include this license 
+ *   notice and a URL through which recipients can access the Corresponding 
+ *   Source.
+ */
+
+/* global setTimeout, clearTimeout */
+
+const browserAPI = this.browser || this.chrome;
+
+const IDLE_DELAY = 1000;
+const watchDogs = [];
+let pendingRequests = new Set();
+
+browserAPI.webRequest.onSendHeaders.addListener(onRequest, { urls: ["<all_urls>"] }, []);
+browserAPI.webRequest.onResponseStarted.addListener(onResponse, { urls: ["<all_urls>"] }, []);
+browserAPI.webRequest.onErrorOccurred.addListener(onResponse, { urls: ["<all_urls>"] });
+
+function onRequest(details) {
+	if (details.tabId != -1) {
+		pendingRequests.add(details.requestId);
+		if (pendingRequests.size > 2) {
+			clearTimeout(watchDogs[2]);
+		}
+		clearTimeout(watchDogs[0]);
+	}
+}
+
+function onResponse(details) {
+	if (details.tabId != -1) {
+		pendingRequests.delete(details.requestId);
+		if (pendingRequests.size == 2) {
+			maybeIdle(2, details.tabId);
+		}
+		if (pendingRequests.size == 0) {
+			maybeIdle(0, details.tabId);
+		}
+	}
+}
+
+function maybeIdle(idleLevel, tabId) {
+	clearTimeout(watchDogs[idleLevel]);
+	watchDogs[idleLevel] = setTimeout(() => browserAPI.tabs.sendMessage(tabId, "network-idle-" + idleLevel), IDLE_DELAY);
+}

package/back-ends/extensions/network-idle/content.js

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2010-2020 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ * 
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or 
+ *   modify it under the terms of the GNU Affero General Public License 
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ * 
+ *   The code in this file is distributed in the hope that it will be useful, 
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may 
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
+ *   AGPL normally required by section 4, provided you include this license 
+ *   notice and a URL through which recipients can access the Corresponding 
+ *   Source.
+ */
+
+/* global window, CustomEvent */
+
+const browserAPI = this.browser || this.chrome;
+const dispatchEvent = event => { try { window.dispatchEvent(event); } catch (error) {  /* ignored */ } };
+
+browserAPI.runtime.onMessage.addListener(message => dispatchEvent(new CustomEvent("single-file-" + message)));

package/back-ends/extensions/network-idle/manifest.json

@@ -0,0 +1,31 @@
+{
+	"name": "network-idle",
+	"version": "0.0.2",
+	"background": {
+		"scripts": [
+			"bg.js"
+		]
+	},
+	"content_scripts": [
+		{
+			"matches": [
+				"<all_urls>"
+			],
+			"run_at": "document_start",
+			"js": [
+				"content.js"
+			]
+		}
+	],
+	"permissions": [
+		"tabs",
+		"webRequest",
+		"<all_urls>"
+	],
+	"browser_specific_settings": {
+		"gecko": {
+			"id": "{caf9306a-8951-4f0c-beb0-bab690d00caf}"
+		}
+	},
+	"manifest_version": 2
+}

package/back-ends/jsdom.js

@@ -0,0 +1,175 @@
+/*
+ * Copyright 2010-2020 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ * 
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or 
+ *   modify it under the terms of the GNU Affero General Public License 
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ * 
+ *   The code in this file is distributed in the hope that it will be useful, 
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may 
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
+ *   AGPL normally required by section 4, provided you include this license 
+ *   notice and a URL through which recipients can access the Corresponding 
+ *   Source.
+ */
+
+/* global require, exports, Buffer, setTimeout */
+
+const crypto = require("crypto");
+
+const jsdom = require("jsdom");
+const { JSDOM, VirtualConsole } = jsdom;
+const iconv = require("iconv-lite");
+
+exports.initialize = async () => { };
+
+exports.getPageData = async options => {
+	let win;
+	try {
+		const dom = await JSDOM.fromURL(options.url, await getBrowserOptions(options));
+		win = dom.window;
+		return await getPageData(win, options);
+	} finally {
+		if (win) {
+			win.close();
+		}
+	}
+};
+
+exports.closeBrowser = () => { };
+
+async function getPageData(win, options) {
+	const doc = win.document;
+	const scripts = await require("./common/scripts.js").get(options);
+	win.TextDecoder = class {
+		constructor(utfLabel) {
+			this.utfLabel = utfLabel;
+		}
+		decode(buffer) {
+			return iconv.decode(Buffer.from(buffer), this.utfLabel);
+		}
+	};
+	win.crypto = {
+		subtle: {
+			digest: async function digestText(algo, text) {
+				const hash = crypto.createHash(algo.replace("-", "").toLowerCase());
+				hash.update(text, "utf-8");
+				return hash.digest();
+			}
+		}
+	};
+	win.Element.prototype.getBoundingClientRect = undefined;
+	win.getComputedStyle = () => { };
+	win.eval(scripts);
+	if (win.document.readyState == "loading" || win.document.readyState == "interactive") {
+		await new Promise(resolve => win.onload = resolve);
+	}
+	executeFrameScripts(doc, scripts);
+	options.removeHiddenElements = false;
+	options.loadDeferredImages = false;
+	if (options.browserWaitDelay) {
+		await new Promise(resolve => setTimeout(resolve, options.browserWaitDelay));
+	}
+	const pageData = await win.singlefile.getPageData(options, { fetch: url => fetchResource(url, options) }, doc, win);
+	if (options.includeInfobar) {
+		await win.infobar.includeScript(pageData);
+	}
+	return pageData;
+
+	async function fetchResource(resourceURL) {
+		return new Promise((resolve, reject) => {
+			const xhrRequest = new win.XMLHttpRequest();
+			xhrRequest.withCredentials = true;
+			xhrRequest.responseType = "arraybuffer";
+			xhrRequest.onerror = event => reject(new Error(event.detail));
+			xhrRequest.onreadystatechange = () => {
+				if (xhrRequest.readyState == win.XMLHttpRequest.DONE) {
+					resolve({
+						arrayBuffer: async () => new Uint8Array(xhrRequest.response).buffer,
+						headers: {
+							get: headerName => xhrRequest.getResponseHeader(headerName)
+						},
+						status: xhrRequest.status
+					});
+				}
+			};
+			xhrRequest.open("GET", resourceURL, true);
+			xhrRequest.send();
+		});
+	}
+}
+
+async function getBrowserOptions(options) {
+	class ResourceLoader extends jsdom.ResourceLoader {
+		_getRequestOptions(fetchOptions) {
+			const requestOptions = super._getRequestOptions(fetchOptions);
+			if (options.httpHeaders) {
+				requestOptions.headers = Object.assign(requestOptions.headers, options.httpHeaders);
+			}
+			return requestOptions;
+		}
+	}
+	const resourceLoader = new ResourceLoader({
+		userAgent: options.userAgent
+	});
+	const jsdomOptions = {
+		virtualConsole: new VirtualConsole(),
+		userAgent: options.userAgent,
+		pretendToBeVisual: true,
+		runScripts: "outside-only",
+		resources: resourceLoader
+	};
+	if (options.browserWidth && options.browserHeight) {
+		jsdomOptions.beforeParse = function (window) {
+			window.outerWidth = window.innerWidth = options.browserWidth;
+			window.outerHeight = window.innerHeight = options.browserHeight;
+		};
+	}
+	if (options.browserCookies && options.browserCookies.length) {
+		jsdomOptions.cookieJar = new jsdom.CookieJar();
+		await Promise.all(options.browserCookies.map(cookie => {
+			let cookieString = cookie.name + "=" + cookie.value;
+			if (cookie.path) {
+				cookieString += ";path=" + cookie.path;
+			}
+			if (cookie.domain) {
+				cookieString += ";domain=" + cookie.domain;
+			}
+			if (cookie.expires) {
+				cookieString += ";max-age=" + cookie.expires;
+			}
+			if (cookie.secure) {
+				cookieString += ";secure";
+			}
+			if (cookie.sameSite) {
+				cookieString += ";samesite=" + options.sameSite;
+			}
+			const cookieOptions = {
+				http: Boolean(cookie.httpOnly)
+			};
+			return new Promise((resolve, reject) => jsdomOptions.cookieJar.setCookie(cookieString, options.url, cookieOptions, error => error ? reject(error) : resolve()));
+		}));
+	}
+	return jsdomOptions;
+}
+
+function executeFrameScripts(doc, scripts) {
+	const frameElements = doc.querySelectorAll("iframe, frame");
+	frameElements.forEach(frameElement => {
+		try {
+			frameElement.contentWindow.Element.prototype.getBoundingClientRect = undefined;
+			frameElement.contentWindow.eval(scripts);
+			executeFrameScripts(frameElement.contentDocument, scripts);
+		} catch (error) {
+			// ignored
+		}
+	});
+}

package/back-ends/playwright-chromium.js

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2010-2020 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ * 
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or 
+ *   modify it under the terms of the GNU Affero General Public License 
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ * 
+ *   The code in this file is distributed in the hope that it will be useful, 
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may 
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
+ *   AGPL normally required by section 4, provided you include this license 
+ *   notice and a URL through which recipients can access the Corresponding 
+ *   Source.
+ */
+
+/* global singlefile, infobar, require, exports */
+
+const playwright = require("playwright").chromium;
+const scripts = require("./common/scripts.js");
+
+const NETWORK_IDLE_STATE = "networkidle";
+
+let browser;
+
+exports.initialize = async options => {
+	browser = await playwright.launch(getBrowserOptions(options));
+};
+
+exports.getPageData = async options => {
+	let page, context;
+	try {
+		context = await browser.newContext({
+			bypassCSP: options.browserBypassCSP === undefined || options.browserBypassCSP
+		});
+		await setContextOptions(context, options);
+		page = await context.newPage();
+		await setPageOptions(page, options);
+		return await getPageData(page, options);
+	} finally {
+		if (page) {
+			await page.close();
+		}
+	}
+};
+
+exports.closeBrowser = () => {
+	if (browser) {
+		return browser.close();
+	}
+};
+
+function getBrowserOptions(options) {
+	const browserOptions = {};
+	if (options.browserHeadless !== undefined) {
+		browserOptions.headless = options.browserHeadless && !options.browserDebug;
+	}
+	browserOptions.args = options.browserArgs ? JSON.parse(options.browserArgs) : [];
+	if (options.browserExecutablePath) {
+		browserOptions.executablePath = options.browserExecutablePath || "chrome";
+	}
+	return browserOptions;
+}
+
+async function setContextOptions(context, options) {
+	if (options.browserCookies && options.browserCookies.length) {
+		await context.addCookies(options.browserCookies);
+	}
+}
+
+async function setPageOptions(page, options) {
+	if (options.browserWidth && options.browserHeight) {
+		await page.setViewportSize({
+			width: options.browserWidth,
+			height: options.browserHeight
+		});
+	}	
+	if (options.httpHeaders) {
+		page.setExtraHTTPHeaders(options.httpHeaders);
+	}
+	if (options.emulateMediaFeatures) {
+		await page.emulateMediaFeatures(options.emulateMediaFeatures);
+	}
+}
+
+async function getPageData(page, options) {
+	const injectedScript = await scripts.get(options);
+	await page.addInitScript(injectedScript);
+	if (options.browserDebug) {
+		await page.waitForTimeout(3000);
+	}
+	await page.goto(options.url, {
+		timeout: options.browserLoadMaxTime || 0,
+		waitUntil: options.browserWaitUntil && options.browserWaitUntil.startsWith("networkidle") ? NETWORK_IDLE_STATE : options.browserWaitUntil || NETWORK_IDLE_STATE
+	});
+	if (options.browserWaitDelay) {
+		await page.waitForTimeout(options.browserWaitDelay);
+	}
+	return await page.evaluate(async options => {
+		const pageData = await singlefile.getPageData(options);
+		if (options.includeInfobar) {
+			await infobar.includeScript(pageData);
+		}
+		return pageData;
+	}, options);
+}

package/back-ends/playwright-firefox.js

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2010-2020 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ * 
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or 
+ *   modify it under the terms of the GNU Affero General Public License 
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ * 
+ *   The code in this file is distributed in the hope that it will be useful, 
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may 
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
+ *   AGPL normally required by section 4, provided you include this license 
+ *   notice and a URL through which recipients can access the Corresponding 
+ *   Source.
+ */
+
+/* global singlefile, infobar, require, exports */
+
+const playwright = require("playwright").firefox;
+const scripts = require("./common/scripts.js");
+
+const NETWORK_IDLE_STATE = "networkidle";
+
+let browser;
+
+exports.initialize = async options => {
+	browser = await playwright.launch(getBrowserOptions(options));
+};
+
+exports.getPageData = async options => {
+	let page, context;
+	try {
+		context = await browser.newContext({
+			bypassCSP: options.browserBypassCSP === undefined || options.browserBypassCSP
+		});
+		await setContextOptions(context, options);
+		page = await context.newPage();
+		await setPageOptions(page, options);
+		return await getPageData(page, options);
+	} finally {
+		if (page) {
+			await page.close();
+		}
+	}
+};
+
+exports.closeBrowser = () => {
+	if (browser) {
+		return browser.close();
+	}
+};
+
+function getBrowserOptions(options) {
+	const browserOptions = {};
+	if (options.browserHeadless !== undefined) {
+		browserOptions.headless = options.browserHeadless && !options.browserDebug;
+	}
+	browserOptions.args = options.browserArgs ? JSON.parse(options.browserArgs) : [];
+	if (options.browserExecutablePath) {
+		browserOptions.executablePath = options.browserExecutablePath || "firefox";
+	}
+	return browserOptions;
+}
+
+async function setContextOptions(context, options) {
+	if (options.browserCookies && options.browserCookies.length) {
+		await context.addCookies(options.browserCookies);
+	}
+}
+
+async function setPageOptions(page, options) {
+	if (options.browserWidth && options.browserHeight) {
+		await page.setViewportSize({
+			width: options.browserWidth,
+			height: options.browserHeight
+		});
+	}
+	if (options.httpHeaders) {
+		page.setExtraHTTPHeaders(options.httpHeaders);
+	}
+	if (options.emulateMediaFeatures) {
+		await page.emulateMediaFeatures(options.emulateMediaFeatures);
+	}
+}
+
+async function getPageData(page, options) {
+	const injectedScript = await scripts.get(options);
+	await page.addInitScript(injectedScript);
+	if (options.browserDebug) {
+		await page.waitForTimeout(3000);
+	}
+	await page.goto(options.url, {
+		timeout: options.browserLoadMaxTime || 0,
+		waitUntil: options.browserWaitUntil && options.browserWaitUntil.startsWith("networkidle") ? NETWORK_IDLE_STATE : options.browserWaitUntil || NETWORK_IDLE_STATE
+	});
+	if (options.browserWaitDelay) {
+		await page.waitForTimeout(options.browserWaitDelay);
+	}
+	return await page.evaluate(async options => {
+		const pageData = await singlefile.getPageData(options);
+		if (options.includeInfobar) {
+			await infobar.includeScript(pageData);
+		}
+		return pageData;
+	}, options);
+}