single-file-cli 1.1.53 → 2.0.2

package/back-ends/jsdom.js

@@ -1,159 +0,0 @@
-/*
- * Copyright 2010-2020 Gildas Lormeau
- * contact : gildas.lormeau <at> gmail.com
- * 
- * This file is part of SingleFile.
- *
- *   The code in this file is free software: you can redistribute it and/or 
- *   modify it under the terms of the GNU Affero General Public License 
- *   (GNU AGPL) as published by the Free Software Foundation, either version 3
- *   of the License, or (at your option) any later version.
- * 
- *   The code in this file is distributed in the hope that it will be useful, 
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
- *   General Public License for more details.
- *
- *   As additional permission under GNU AGPL version 3 section 7, you may 
- *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
- *   AGPL normally required by section 4, provided you include this license 
- *   notice and a URL through which recipients can access the Corresponding 
- *   Source.
- */
-
-/* global require, exports, Buffer, setTimeout */
-
-const crypto = require("crypto");
-
-const jsdom = require("jsdom");
-const { JSDOM, VirtualConsole } = jsdom;
-const iconv = require("iconv-lite");
-
-exports.initialize = async () => { };
-
-exports.getPageData = async options => {
-	let win;
-	try {
-		const dom = await JSDOM.fromURL(options.url, await getBrowserOptions(options));
-		win = dom.window;
-		return await getPageData(win, options);
-	} finally {
-		if (win) {
-			win.close();
-		}
-	}
-};
-
-exports.closeBrowser = () => { };
-
-async function getPageData(win, options) {
-	const doc = win.document;
-	const scripts = await require("./common/scripts.js").get(options);
-	win.TextDecoder = class {
-		constructor(utfLabel) {
-			this.utfLabel = utfLabel;
-		}
-		decode(buffer) {
-			return iconv.decode(Buffer.from(buffer), this.utfLabel);
-		}
-	};
-	win.TextEncoder = class {
-		encode(value) {
-			return iconv.encode(value, "utf-8");
-		}
-	};
-	win.crypto = {
-		subtle: {
-			digest: async function digestText(algo, text) {
-				const hash = crypto.createHash(algo.replace("-", "").toLowerCase());
-				hash.update(text, "utf-8");
-				return hash.digest();
-			}
-		}
-	};
-	win.Element.prototype.getBoundingClientRect = undefined;
-	win.getComputedStyle = () => { };
-	win.eval(scripts);
-	if (win.document.readyState == "loading" || win.document.readyState == "interactive") {
-		await new Promise(resolve => win.onload = resolve);
-	}
-	executeFrameScripts(doc, scripts);
-	options.removeHiddenElements = false;
-	options.loadDeferredImages = false;
-	if (options.browserWaitDelay) {
-		await new Promise(resolve => setTimeout(resolve, options.browserWaitDelay));
-	}
-	const pageData = await win.singlefile.getPageData(options, undefined, doc, win);
-	if (options.compressContent) {
-		pageData.content = new Uint8Array(pageData.content);
-	}
-	return pageData;
-}
-
-async function getBrowserOptions(options) {
-	class ResourceLoader extends jsdom.ResourceLoader {
-		_getRequestOptions(fetchOptions) {
-			const requestOptions = super._getRequestOptions(fetchOptions);
-			if (options.httpHeaders) {
-				requestOptions.headers = Object.assign(requestOptions.headers, options.httpHeaders);
-			}
-			return requestOptions;
-		}
-	}
-	const resourceLoader = new ResourceLoader({
-		userAgent: options.userAgent,
-		strictSSL: options.browserIgnoreInsecureCerts === undefined || !options.browserIgnoreInsecureCerts
-	});
-	const jsdomOptions = {
-		virtualConsole: new VirtualConsole(),
-		userAgent: options.userAgent,
-		pretendToBeVisual: true,
-		runScripts: "outside-only",
-		resources: resourceLoader
-	};
-	if (options.browserWidth && options.browserHeight) {
-		jsdomOptions.beforeParse = function (window) {
-			window.outerWidth = window.innerWidth = options.browserWidth;
-			window.outerHeight = window.innerHeight = options.browserHeight;
-		};
-	}
-	if (options.browserCookies && options.browserCookies.length) {
-		jsdomOptions.cookieJar = new jsdom.CookieJar();
-		await Promise.all(options.browserCookies.map(cookie => {
-			let cookieString = cookie.name + "=" + cookie.value;
-			if (cookie.path) {
-				cookieString += ";path=" + cookie.path;
-			}
-			if (cookie.domain) {
-				cookieString += ";domain=" + cookie.domain;
-			}
-			if (cookie.expires) {
-				cookieString += ";max-age=" + cookie.expires;
-			}
-			if (cookie.secure) {
-				cookieString += ";secure";
-			}
-			if (cookie.sameSite) {
-				cookieString += ";samesite=" + options.sameSite;
-			}
-			const cookieOptions = {
-				http: Boolean(cookie.httpOnly)
-			};
-			return new Promise((resolve, reject) => jsdomOptions.cookieJar.setCookie(cookieString, options.url, cookieOptions, error => error ? reject(error) : resolve()));
-		}));
-	}
-	return jsdomOptions;
-}
-
-function executeFrameScripts(doc, scripts) {
-	const frameElements = doc.querySelectorAll("iframe, frame");
-	frameElements.forEach(frameElement => {
-		try {
-			frameElement.contentWindow.Element.prototype.getBoundingClientRect = undefined;
-			frameElement.contentWindow.eval(scripts);
-			executeFrameScripts(frameElement.contentDocument, scripts);
-		} catch (error) {
-			// ignored
-		}
-	});
-}

package/back-ends/playwright-chromium.js

@@ -1,126 +0,0 @@
-/*
- * Copyright 2010-2020 Gildas Lormeau
- * contact : gildas.lormeau <at> gmail.com
- * 
- * This file is part of SingleFile.
- *
- *   The code in this file is free software: you can redistribute it and/or 
- *   modify it under the terms of the GNU Affero General Public License 
- *   (GNU AGPL) as published by the Free Software Foundation, either version 3
- *   of the License, or (at your option) any later version.
- * 
- *   The code in this file is distributed in the hope that it will be useful, 
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
- *   General Public License for more details.
- *
- *   As additional permission under GNU AGPL version 3 section 7, you may 
- *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
- *   AGPL normally required by section 4, provided you include this license 
- *   notice and a URL through which recipients can access the Corresponding 
- *   Source.
- */
-
-/* global singlefile, require, exports */
-
-const playwright = require("playwright").chromium;
-const scripts = require("./common/scripts.js");
-
-const NETWORK_IDLE_STATE = "networkidle";
-
-let browser;
-
-exports.initialize = async options => {
-	browser = await playwright.launch(getBrowserOptions(options));
-};
-
-exports.getPageData = async options => {
-	let page, context;
-	try {
-		const contextOptions = {
-			bypassCSP: options.browserBypassCSP === undefined || options.browserBypassCSP,
-			ignoreHTTPSErrors: options.browserIgnoreInsecureCerts !== undefined && options.browserIgnoreInsecureCerts
-		};
-		if (options.httpProxyServer) {
-			contextOptions.proxy = {
-				server: options.httpProxyServer
-			};
-			if (options.httpProxyUsername) {
-				contextOptions.proxy.username = options.httpProxyUsername;
-			}
-			if (options.httpProxyPassword) {
-				contextOptions.proxy.password = options.httpProxyPassword;
-			}
-		}
-		context = await browser.newContext(contextOptions);
-		await setContextOptions(context, options);
-		page = await context.newPage();
-		await setPageOptions(page, options);
-		return await getPageData(page, options);
-	} finally {
-		if (page && !options.browserDebug) {
-			await page.close();
-		}
-	}
-};
-
-exports.closeBrowser = () => {
-	if (browser) {
-		return browser.close();
-	}
-};
-
-function getBrowserOptions(options) {
-	const browserOptions = {};
-	if (options.browserHeadless !== undefined) {
-		browserOptions.headless = options.browserHeadless && !options.browserDebug;
-	}
-	browserOptions.args = options.browserArgs ? JSON.parse(options.browserArgs) : [];
-	if (options.browserExecutablePath) {
-		browserOptions.executablePath = options.browserExecutablePath || "chrome";
-	}
-	return browserOptions;
-}
-
-async function setContextOptions(context, options) {
-	if (options.browserCookies && options.browserCookies.length) {
-		await context.addCookies(options.browserCookies);
-	}
-}
-
-async function setPageOptions(page, options) {
-	if (options.browserWidth && options.browserHeight) {
-		await page.setViewportSize({
-			width: options.browserWidth,
-			height: options.browserHeight
-		});
-	}
-	if (options.httpHeaders) {
-		page.setExtraHTTPHeaders(options.httpHeaders);
-	}
-	if (options.emulateMediaFeatures) {
-		await page.emulateMedia({
-			features: options.emulateMediaFeatures
-		});
-	}
-}
-
-async function getPageData(page, options) {
-	const injectedScript = await scripts.get(options);
-	await page.addInitScript(injectedScript);
-	if (options.browserDebug) {
-		await page.waitForTimeout(3000);
-	}
-	await page.goto(options.url, {
-		timeout: options.browserLoadMaxTime || 0,
-		waitUntil: options.browserWaitUntil && options.browserWaitUntil.startsWith("networkidle") ? NETWORK_IDLE_STATE : options.browserWaitUntil || NETWORK_IDLE_STATE
-	});
-	if (options.browserWaitDelay) {
-		await page.waitForTimeout(options.browserWaitDelay);
-	}
-	const pageData = await page.evaluate(async options => await singlefile.getPageData(options), options);
-	if (options.compressContent) {
-		pageData.content = new Uint8Array(pageData.content);
-	}
-	return pageData;
-}

package/back-ends/playwright-firefox.js

@@ -1,115 +0,0 @@
-/*
- * Copyright 2010-2020 Gildas Lormeau
- * contact : gildas.lormeau <at> gmail.com
- * 
- * This file is part of SingleFile.
- *
- *   The code in this file is free software: you can redistribute it and/or 
- *   modify it under the terms of the GNU Affero General Public License 
- *   (GNU AGPL) as published by the Free Software Foundation, either version 3
- *   of the License, or (at your option) any later version.
- * 
- *   The code in this file is distributed in the hope that it will be useful, 
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
- *   General Public License for more details.
- *
- *   As additional permission under GNU AGPL version 3 section 7, you may 
- *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
- *   AGPL normally required by section 4, provided you include this license 
- *   notice and a URL through which recipients can access the Corresponding 
- *   Source.
- */
-
-/* global singlefile, require, exports */
-
-const playwright = require("playwright").firefox;
-const scripts = require("./common/scripts.js");
-
-const NETWORK_IDLE_STATE = "networkidle";
-
-let browser;
-
-exports.initialize = async options => {
-	browser = await playwright.launch(getBrowserOptions(options));
-};
-
-exports.getPageData = async options => {
-	let page, context;
-	try {
-		context = await browser.newContext({
-			bypassCSP: options.browserBypassCSP === undefined || options.browserBypassCSP,
-			ignoreHTTPSErrors: options.browserIgnoreInsecureCerts !== undefined && options.browserIgnoreInsecureCerts
-		});
-		await setContextOptions(context, options);
-		page = await context.newPage();
-		await setPageOptions(page, options);
-		return await getPageData(page, options);
-	} finally {
-		if (page && !options.browserDebug) {
-			await page.close();
-		}
-	}
-};
-
-exports.closeBrowser = () => {
-	if (browser) {
-		return browser.close();
-	}
-};
-
-function getBrowserOptions(options) {
-	const browserOptions = {};
-	if (options.browserHeadless !== undefined) {
-		browserOptions.headless = options.browserHeadless && !options.browserDebug;
-	}
-	browserOptions.args = options.browserArgs ? JSON.parse(options.browserArgs) : [];
-	if (options.browserExecutablePath) {
-		browserOptions.executablePath = options.browserExecutablePath || "firefox";
-	}
-	return browserOptions;
-}
-
-async function setContextOptions(context, options) {
-	if (options.browserCookies && options.browserCookies.length) {
-		await context.addCookies(options.browserCookies);
-	}
-}
-
-async function setPageOptions(page, options) {
-	if (options.browserWidth && options.browserHeight) {
-		await page.setViewportSize({
-			width: options.browserWidth,
-			height: options.browserHeight
-		});
-	}
-	if (options.httpHeaders) {
-		page.setExtraHTTPHeaders(options.httpHeaders);
-	}
-	if (options.emulateMediaFeatures) {
-		await page.emulateMedia({
-			features: options.emulateMediaFeatures
-		});
-	}
-	options.browserWaitUntil = "load";
-}
-
-async function getPageData(page, options) {
-	const injectedScript = await scripts.get(options);
-	await page.addInitScript(injectedScript);
-	if (options.browserDebug) {
-		await page.waitForTimeout(3000);
-	}
-	await page.goto(options.url, {
-		timeout: options.browserLoadMaxTime || 0,
-		waitUntil: options.browserWaitUntil && options.browserWaitUntil.startsWith("networkidle") ? NETWORK_IDLE_STATE : options.browserWaitUntil || NETWORK_IDLE_STATE
-	});
-	if (options.browserWaitDelay) {
-		await page.waitForTimeout(options.browserWaitDelay);
-	}
-	const pageData = await page.evaluate(async options => await singlefile.getPageData(options), options);
-	if (options.compressContent) {
-		pageData.content = new Uint8Array(pageData.content);
-	}
-	return pageData;
-}

package/back-ends/playwright-webkit.js

@@ -1,126 +0,0 @@
-/*
- * Copyright 2010-2020 Gildas Lormeau
- * contact : gildas.lormeau <at> gmail.com
- * 
- * This file is part of SingleFile.
- *
- *   The code in this file is free software: you can redistribute it and/or 
- *   modify it under the terms of the GNU Affero General Public License 
- *   (GNU AGPL) as published by the Free Software Foundation, either version 3
- *   of the License, or (at your option) any later version.
- * 
- *   The code in this file is distributed in the hope that it will be useful, 
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
- *   General Public License for more details.
- *
- *   As additional permission under GNU AGPL version 3 section 7, you may 
- *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
- *   AGPL normally required by section 4, provided you include this license 
- *   notice and a URL through which recipients can access the Corresponding 
- *   Source.
- */
-
-/* global singlefile, require, exports */
-
-const playwright = require("playwright").webkit;
-const scripts = require("./common/scripts.js");
-
-const NETWORK_IDLE_STATE = "networkidle";
-
-let browser;
-
-exports.initialize = async options => {
-	browser = await playwright.launch(getBrowserOptions(options));
-};
-
-exports.getPageData = async options => {
-	let page, context;
-	try {
-		const contextOptions = {
-			bypassCSP: options.browserBypassCSP === undefined || options.browserBypassCSP,
-			ignoreHTTPSErrors: options.browserIgnoreInsecureCerts !== undefined && options.browserIgnoreInsecureCerts
-		};
-		if (options.httpProxyServer) {
-			contextOptions.proxy = {
-				server: options.httpProxyServer
-			};
-			if (options.httpProxyUsername) {
-				contextOptions.proxy.username = options.httpProxyUsername;
-			}
-			if (options.httpProxyPassword) {
-				contextOptions.proxy.password = options.httpProxyPassword;
-			}
-		}
-		context = await browser.newContext(contextOptions);
-		await setContextOptions(context, options);
-		page = await context.newPage();
-		await setPageOptions(page, options);
-		return await getPageData(page, options);
-	} finally {
-		if (page && !options.browserDebug) {
-			await page.close();
-		}
-	}
-};
-
-exports.closeBrowser = () => {
-	if (browser) {
-		return browser.close();
-	}
-};
-
-function getBrowserOptions(options) {
-	const browserOptions = {};
-	if (options.browserHeadless !== undefined) {
-		browserOptions.headless = options.browserHeadless && !options.browserDebug;
-	}
-	browserOptions.args = options.browserArgs ? JSON.parse(options.browserArgs) : [];
-	if (options.browserExecutablePath) {
-		browserOptions.executablePath = options.browserExecutablePath || "safari";
-	}
-	return browserOptions;
-}
-
-async function setContextOptions(context, options) {
-	if (options.browserCookies && options.browserCookies.length) {
-		await context.addCookies(options.browserCookies);
-	}
-}
-
-async function setPageOptions(page, options) {
-	if (options.browserWidth && options.browserHeight) {
-		await page.setViewportSize({
-			width: options.browserWidth,
-			height: options.browserHeight
-		});
-	}
-	if (options.httpHeaders) {
-		page.setExtraHTTPHeaders(options.httpHeaders);
-	}
-	if (options.emulateMediaFeatures) {
-		await page.emulateMedia({
-			features: options.emulateMediaFeatures
-		});
-	}
-}
-
-async function getPageData(page, options) {
-	const injectedScript = await scripts.get(options);
-	await page.addInitScript(injectedScript);
-	if (options.browserDebug) {
-		await page.waitForTimeout(3000);
-	}
-	await page.goto(options.url, {
-		timeout: options.browserLoadMaxTime || 0,
-		waitUntil: options.browserWaitUntil && options.browserWaitUntil.startsWith("networkidle") ? NETWORK_IDLE_STATE : options.browserWaitUntil || NETWORK_IDLE_STATE
-	});
-	if (options.browserWaitDelay) {
-		await page.waitForTimeout(options.browserWaitDelay);
-	}
-	const pageData = await page.evaluate(async options => await singlefile.getPageData(options), options);
-	if (options.compressContent) {
-		pageData.content = new Uint8Array(pageData.content);
-	}
-	return pageData;
-}

package/back-ends/puppeteer-firefox.js

@@ -1,172 +0,0 @@
-/*
- * Copyright 2010-2020 Gildas Lormeau
- * contact : gildas.lormeau <at> gmail.com
- * 
- * This file is part of SingleFile.
- *
- *   The code in this file is free software: you can redistribute it and/or 
- *   modify it under the terms of the GNU Affero General Public License 
- *   (GNU AGPL) as published by the Free Software Foundation, either version 3
- *   of the License, or (at your option) any later version.
- * 
- *   The code in this file is distributed in the hope that it will be useful, 
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of 
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero 
- *   General Public License for more details.
- *
- *   As additional permission under GNU AGPL version 3 section 7, you may 
- *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU 
- *   AGPL normally required by section 4, provided you include this license 
- *   notice and a URL through which recipients can access the Corresponding 
- *   Source.
- */
-
-/* global singlefile, require, exports, setTimeout */
-
-const puppeteer = require("puppeteer-core");
-const scripts = require("./common/scripts.js");
-
-const EXECUTION_CONTEXT_DESTROYED_ERROR = "Execution context was destroyed";
-const NETWORK_IDLE_STATE = "networkidle0";
-
-let browser;
-
-exports.initialize = async options => {
-	browser = await puppeteer.launch(getBrowserOptions(options));
-};
-
-exports.getPageData = async options => {
-	let page;
-	try {
-		page = await browser.newPage();
-		await setPageOptions(page, options);
-		return await getPageData(browser, page, options);
-	} finally {
-		if (page && !options.browserDebug) {
-			await page.close();
-		}
-	}
-};
-
-exports.closeBrowser = () => {
-	if (browser) {
-		return browser.close();
-	}
-};
-
-function getBrowserOptions(options) {
-	const browserOptions = {};
-	if (options.browserHeadless !== undefined) {
-		browserOptions.headless = options.browserHeadless && !options.browserDebug;
-	}
-	browserOptions.args = options.browserArgs ? JSON.parse(options.browserArgs) : [];
-	if (options.browserExecutablePath) {
-		browserOptions.executablePath = options.browserExecutablePath || "firefox";
-	}
-	if (options.browserIgnoreInsecureCerts !== undefined) {
-		browserOptions.ignoreHTTPSErrors = options.browserIgnoreInsecureCerts;
-	}
-	browserOptions.product = "firefox";
-	return browserOptions;
-}
-
-async function setPageOptions(page, options) {
-	if (options.browserWidth && options.browserHeight) {
-		await page.setViewport({
-			width: options.browserWidth,
-			height: options.browserHeight
-		});
-	}
-	if ((options.browserBypassCSP === undefined || options.browserBypassCSP) && page.setBypassCSP) {
-		try {
-			await page.setBypassCSP(true);
-		} catch (error) {
-			// ignored
-		}
-	}
-	if (options.httpHeaders) {
-		try {
-			await page.setExtraHTTPHeaders(options.httpHeaders);
-		} catch (error) {
-			// ignored
-		}
-	}
-	if (options.browserCookies && options.browserCookies.length) {
-		await page.setCookie(...options.browserCookies);
-	}
-	if (options.emulateMediaFeatures) {
-		try {
-			await page.emulateMediaFeatures(options.emulateMediaFeatures);
-		} catch (error) {
-			// ignored
-		}
-	}
-	options.browserWaitUntil = "load";
-}
-
-async function getPageData(browser, page, options) {
-	const injectedScript = await scripts.get(options);
-	await page.evaluateOnNewDocument(injectedScript);
-	if (options.browserDebug) {
-		await new Promise(resolve => setTimeout(resolve, 3000));
-	}
-	await pageGoto(page, options);
-	try {
-		await page.evaluate(injectedScript);
-		if (options.browserWaitDelay) {
-			await new Promise(resolve => setTimeout(resolve, options.browserWaitDelay));
-		}
-		const pageData = await page.evaluate(async options => await singlefile.getPageData(options), options);
-		if (options.compressContent) {
-			pageData.content = new Uint8Array(pageData.content);
-		}
-		return pageData;
-	} catch (error) {
-		if (error.message && error.message.includes(EXECUTION_CONTEXT_DESTROYED_ERROR)) {
-			const pageData = await handleJSRedirect(browser, options);
-			if (pageData) {
-				return pageData;
-			} else {
-				throw error;
-			}
-		} else if (error.name != "TimeoutError") {
-			throw error;
-		}
-	}
-}
-
-async function handleJSRedirect(browser, options) {
-	const pages = await browser.pages();
-	const page = pages[1] || pages[0];
-	try {
-		await pageGoto(page, options);
-	} catch (error) {
-		if (error.name != "TimeoutError") {
-			throw error;
-		}
-	}
-	const url = page.url();
-	if (url != options.url) {
-		options.url = url;
-		await browser.close();
-		return exports.getPageData(options);
-	}
-}
-
-async function pageGoto(page, options) {
-	try {
-		await page.goto(options.url, {
-			timeout: options.browserLoadMaxTime || 0,
-			waitUntil: options.browserWaitUntil || NETWORK_IDLE_STATE
-		});
-	} catch (error) {
-		if (error.message.includes("Unknown waitUntil condition")) {
-			await page.goto(options.url, {
-				timeout: options.browserLoadMaxTime || 0,
-				waitUntil: "load"
-			});
-		} else {
-			throw error;
-		}
-	}
-}