single-file-cli 1.0.26 → 1.0.28

package/args.js

@@ -67,6 +67,9 @@ const args = require("yargs")
 		"group-duplicate-images": true,
 		"max-size-duplicate-images": 512 * 1024,
 		"http-header": [],
+		"http-proxy-server": "",
+		"http-proxy-username": "",
+		"http-proxy-password": "",
 		"include-infobar": false,
 		"insert-meta-csp": true,
 		"load-deferred-images": true,
@@ -195,6 +198,12 @@ const args = require("yargs")
 	.number("max-size-duplicate-images")
 	.options("http-header", { description: "Extra HTTP header (puppeteer, jsdom)" })
 	.array("http-header")
+	.options("http-proxy-server", { description: "Proxy address (puppeteer)" })
+	.string("http-proxy-server")
+	.options("http-proxy-username", { description: "HTTP username (puppeteer)" })
+	.string("http-proxy-username")
+	.options("http-proxy-password", { description: "HTTP password (puppeteer)" })
+	.string("http-proxy-password")
 	.options("include-BOM", { description: "Include the UTF-8 BOM into the HTML page" })
 	.boolean("include-BOM")
 	.options("include-infobar", { description: "Include the infobar" })
@@ -259,6 +268,7 @@ args.includeBOM = args.includeBom;
 args.crawlReplaceURLs = args.crawlReplaceUrls;
 args.crawlRemoveURLFragment = args.crawlRemoveUrlFragment;
 args.insertMetaCSP = args.insertMetaCsp;
+args.saveOriginalURLs = args.saveOriginalUrls;
 if (args.removeScripts) {
 	args.blockScripts = true;
 }

package/back-ends/playwright-chromium.js

@@ -37,9 +37,21 @@ exports.initialize = async options => {
 exports.getPageData = async options => {
 	let page, context;
 	try {
-		context = await browser.newContext({
+		const contextOptions = {
 			bypassCSP: options.browserBypassCSP === undefined || options.browserBypassCSP
-		});
+		};
+		if (options.httpProxyServer) {
+			contextOptions.proxy = {
+				server: options.httpProxyServer
+			};
+			if (options.httpProxyUsername) {
+				contextOptions.proxy.username = options.httpProxyUsername;
+			}
+			if (options.httpProxyPassword) {
+				contextOptions.proxy.password = options.httpProxyPassword;
+			}
+		}
+		context = await browser.newContext(contextOptions);
 		await setContextOptions(context, options);
 		page = await context.newPage();
 		await setPageOptions(page, options);

package/back-ends/playwright-firefox.js

@@ -90,6 +90,7 @@ async function setPageOptions(page, options) {
 			features: options.emulateMediaFeatures
 		});
 	}
+	options.browserWaitUntil = "load";
 }
 
 async function getPageData(page, options) {

package/back-ends/playwright-webkit.js

@@ -37,9 +37,21 @@ exports.initialize = async options => {
 exports.getPageData = async options => {
 	let page, context;
 	try {
-		context = await browser.newContext({
+		const contextOptions = {
 			bypassCSP: options.browserBypassCSP === undefined || options.browserBypassCSP
-		});
+		};
+		if (options.httpProxyServer) {
+			contextOptions.proxy = {
+				server: options.httpProxyServer
+			};
+			if (options.httpProxyUsername) {
+				contextOptions.proxy.username = options.httpProxyUsername;
+			}
+			if (options.httpProxyPassword) {
+				contextOptions.proxy.password = options.httpProxyPassword;
+			}
+		}
+		context = await browser.newContext(contextOptions);
 		await setContextOptions(context, options);
 		page = await context.newPage();
 		await setPageOptions(page, options);

package/back-ends/puppeteer-firefox.js

@@ -98,6 +98,7 @@ async function setPageOptions(page, options) {
 			// ignored
 		}
 	}
+	options.browserWaitUntil = "load";
 }
 
 async function getPageData(browser, page, options) {

package/back-ends/puppeteer.js

@@ -30,7 +30,7 @@ const EXECUTION_CONTEXT_DESTROYED_ERROR = "Execution context was destroyed";
 const NETWORK_IDLE_STATE = "networkidle0";
 const NETWORK_STATES = ["networkidle0", "networkidle2", "load", "domcontentloaded"];
 
-let browser;
+let browser, context;
 
 exports.initialize = async options => {
 	if (options.browserServer) {
@@ -45,10 +45,15 @@ exports.getPageData = async (options, page) => {
 	const privatePage = !page;
 	try {
 		if (privatePage) {
-			page = await browser.newPage();
+			const contextOptions = {};
+			if (options.httpProxyServer) {
+				contextOptions.proxyServer = options.httpProxyServer;
+			}
+			context = await browser.createIncognitoBrowserContext(contextOptions);
+			page = await context.newPage();
 		}
 		await setPageOptions(page, options);
-		return await getPageData(browser, page, options);
+		return await getPageData(context || browser, page, options);
 	} finally {
 		if (privatePage && !options.browserDebug) {
 			await page.close();
@@ -109,9 +114,15 @@ async function setPageOptions(page, options) {
 	if (options.emulateMediaFeatures) {
 		await page.emulateMediaFeatures(options.emulateMediaFeatures);
 	}
+	if (options.httpProxyServer && (options.httpProxyUsername || options.httpProxyPassword)) {
+		await page.authenticate({
+			username: options.httpProxyUsername,
+			password: options.httpProxyPassword
+		});
+	}
 }
 
-async function getPageData(browser, page, options) {
+async function getPageData(context, page, options) {
 	const injectedScript = await scripts.get(options);
 	await page.evaluateOnNewDocument(injectedScript);
 	if (options.browserDebug) {
@@ -124,7 +135,7 @@ async function getPageData(browser, page, options) {
 			const browserWaitUntil = NETWORK_STATES[(NETWORK_STATES.indexOf(options.browserWaitUntil) + 1)];
 			if (browserWaitUntil) {
 				options.browserWaitUntil = browserWaitUntil;
-				return getPageData(browser, page, options);
+				return getPageData(context, page, options);
 			} else {
 				throw error;
 			}
@@ -141,7 +152,7 @@ async function getPageData(browser, page, options) {
 		}, options);
 	} catch (error) {
 		if (error.message && error.message.includes(EXECUTION_CONTEXT_DESTROYED_ERROR)) {
-			const pageData = await handleJSRedirect(browser, options);
+			const pageData = await handleJSRedirect(context, options);
 			if (pageData) {
 				return pageData;
 			} else {
@@ -153,8 +164,8 @@ async function getPageData(browser, page, options) {
 	}
 }
 
-async function handleJSRedirect(browser, options) {
-	const pages = await browser.pages();
+async function handleJSRedirect(context, options) {
+	const pages = await context.pages();
 	const page = pages[1] || pages[0];
 	try {
 		await pageGoto(page, options);
@@ -166,7 +177,7 @@ async function handleJSRedirect(browser, options) {
 	const url = page.url();
 	if (url != options.url) {
 		options.url = url;
-		await browser.close();
+		await context.close();
 		return exports.getPageData(options);
 	}
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "1.0.26",
+	"version": "1.0.28",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"license": "AGPL-3.0-or-later",