single-file-core 1.6.27 → 1.6.29

package/core/helper.js

@@ -22,6 +22,7 @@
  */
 
 import * as cssUnescape from "./../vendor/css-unescape.js";
+import * as srcsetParser from "./../vendor/html-srcset-parser.js";
 import * as sha from "./lib/sha.js";
 import * as hooksFrames from "./../processors/hooks/content/content-hooks-frames.js";
 import * as infobar from "./infobar.js";
@@ -44,6 +45,7 @@ const ON_AFTER_CAPTURE_EVENT_NAME = SINGLE_FILE_PREFIX + "on-after-capture";
 const GET_ADOPTED_STYLESHEETS_REQUEST_EVENT = SINGLE_FILE_PREFIX + "request-get-adopted-stylesheets";
 const GET_ADOPTED_STYLESHEETS_RESPONSE_EVENT = SINGLE_FILE_PREFIX + "response-get-adopted-stylesheets";
 const UNREGISTER_GET_ADOPTED_STYLESHEETS_REQUEST_EVENT = SINGLE_FILE_PREFIX + "unregister-request-get-adopted-stylesheets";
+const CLEAR_ADOPTED_STYLESHEETS_CACHE_EVENT = SINGLE_FILE_PREFIX + "clear-adopted-stylesheets-cache";
 const ON_INIT_USERSCRIPT_EVENT = SINGLE_FILE_PREFIX + "user-script-init";
 const REMOVED_CONTENT_ATTRIBUTE_NAME = "data-" + SINGLE_FILE_PREFIX + "removed-content";
 const HIDDEN_CONTENT_ATTRIBUTE_NAME = "data-" + SINGLE_FILE_PREFIX + "hidden-content";
@@ -117,6 +119,37 @@ const NESTING_SHADOW_ROOT_TRACK_ID_PREFIX = "s";
 const HTML_NAMESPACE = "http://www.w3.org/1999/xhtml";
 const RAW_TEXT_TAG_NAMES = ["SCRIPT", "STYLE", "TEXTAREA", "TITLE", "XMP", "IFRAME", "NOEMBED", "NOFRAMES", "PLAINTEXT", "NOSCRIPT"];
 const COMMENT_NODE_FILTER = 128;
+const PROTECTED_OPTION_NAMES = [
+	"password",
+	"woleetKey",
+	"saveToGDrive",
+	"saveToDropbox",
+	"saveWithCompanion",
+	"forceWebAuthFlow",
+	"saveWithWebDAV",
+	"webDAVURL",
+	"webDAVUser",
+	"webDAVPassword",
+	"saveWithMCP",
+	"mcpServerUrl",
+	"mcpAuthToken",
+	"saveToGitHub",
+	"githubToken",
+	"githubUser",
+	"githubRepository",
+	"githubBranch",
+	"saveToRestFormApi",
+	"saveToRestFormApiUrl",
+	"saveToRestFormApiToken",
+	"saveToRestFormApiFileFieldName",
+	"saveToRestFormApiUrlFieldName",
+	"saveToS3",
+	"S3Domain",
+	"S3Region",
+	"S3Bucket",
+	"S3AccessKey",
+	"S3SecretKey"
+];
 const DEPRECATED_OPTION_NAMES = {
 	loadDeferredImages: "loadDeferredContent",
 	loadDeferredImagesMaxIdleTime: "loadDeferredContentMaxIdleTime",
@@ -124,7 +157,8 @@ const DEPRECATED_OPTION_NAMES = {
 	loadDeferredImagesBlockStorage: "loadDeferredContentBlockStorage",
 	loadDeferredImagesKeepZoomLevel: "loadDeferredContentKeepZoomLevel",
 	loadDeferredImagesDispatchScrollEvent: "loadDeferredContentDispatchScrollEvent",
-	loadDeferredImagesBeforeFrames: "loadDeferredContentBeforeFrames"
+	loadDeferredImagesBeforeFrames: "loadDeferredContentBeforeFrames",
+	blockAlternativeImages: "removeAlternativeImages"
 };
 const addEventListener = (type, listener, options) => globalThis.addEventListener(type, listener, options);
 const removeEventListener = (type, listener, options) => globalThis.removeEventListener(type, listener, options);
@@ -259,7 +293,7 @@ function onInitUserScript({ detail }) {
 		delete userScriptOptions.frames;
 		delete userScriptOptions.taskId;
 		delete userScriptOptions._migratedTemplateFormat;
-		delete userScriptOptions.woleetKey;
+		PROTECTED_OPTION_NAMES.forEach(name => delete userScriptOptions[name]);
 		let detailUserScript;
 		try {
 			detailUserScript = detail == "jsonDetail" ? JSON.stringify({ options: userScriptOptions }) : { options: userScriptOptions };
@@ -276,6 +310,7 @@ function onInitUserScript({ detail }) {
 				try {
 					const detail = typeof event.detail == "string" ? JSON.parse(event.detail) : event.detail;
 					if (detail.options) {
+						PROTECTED_OPTION_NAMES.forEach(name => delete detail.options[name]);
 						Object.assign(options, detail.options);
 					}
 					// eslint-disable-next-line no-unused-vars
@@ -324,6 +359,12 @@ function preProcessDoc(doc, win, options) {
 	let elementsInfo;
 	if (win && doc.documentElement) {
 		markInvalidNesting(doc);
+		try {
+			doc.dispatchEvent(new CustomEvent(CLEAR_ADOPTED_STYLESHEETS_CACHE_EVENT));
+			// eslint-disable-next-line no-unused-vars
+		} catch (error) {
+			// ignored
+		}
 		elementsInfo = getElementsInfo(win, doc, doc.documentElement, options);
 		if (options.removeUnusedFonts && doc.defaultView) {
 			getRootElementUsedFonts(win, doc.documentElement, elementsInfo);
@@ -824,7 +865,7 @@ function getElementsInfo(win, doc, element, options, data = { usedFonts: new Map
 				shadowRootInfo.clonable = shadowRoot.clonable;
 				shadowRootInfo.serializable = shadowRoot.serializable;
 				try {
-					if (shadowRoot.adoptedStyleSheets && shadowRoot.adoptedStyleSheets.length === undefined) {
+					if (shadowRoot.adoptedStyleSheets) {
 						shadowRoot.dispatchEvent(new CustomEvent(UNREGISTER_GET_ADOPTED_STYLESHEETS_REQUEST_EVENT, { bubbles: true }));
 					}
 					// eslint-disable-next-line no-unused-vars
@@ -908,6 +949,9 @@ function getResourcesInfo(win, doc, element, options, data, elementHidden, compu
 				EMPTY_RESOURCE :
 				(options.loadDeferredContent && element.getAttribute(LAZY_SRC_ATTRIBUTE_NAME)) || element.currentSrc
 		};
+		if (!elementHidden && element.parentElement && element.parentElement.tagName.toUpperCase() == "PICTURE") {
+			imageData.sourceSize = getSourceSize(win, element.parentElement, imageData.currentSrc);
+		}
 		data.images.push(imageData);
 		element.setAttribute(IMAGE_ATTRIBUTE_NAME, data.images.length - 1);
 		data.markedElements.push(element);
@@ -1241,6 +1285,42 @@ function getStylesheetsData(doc, markedElements) {
 	}
 }
 
+function getSourceSize(win, pictureElement, currentSrc) {
+	if (currentSrc) {
+		const baseURI = pictureElement.ownerDocument.baseURI;
+		const sourceElement = Array.from(pictureElement.querySelectorAll(":scope > source")).find(sourceElement => {
+			const media = sourceElement.getAttribute("media");
+			return (!media || win.matchMedia(media).matches) && getSrcsetURLs(sourceElement.getAttribute("srcset"), baseURI).includes(currentSrc);
+		});
+		if (sourceElement && (sourceElement.hasAttribute("width") || sourceElement.hasAttribute("height"))) {
+			return {
+				width: sourceElement.getAttribute("width"),
+				height: sourceElement.getAttribute("height")
+			};
+		}
+	}
+}
+
+function getSrcsetURLs(srcset, baseURI) {
+	const urls = [];
+	if (srcset) {
+		try {
+			srcsetParser.process(srcset).forEach(candidate => {
+				try {
+					urls.push(new URL(candidate.url, baseURI).href);
+					// eslint-disable-next-line no-unused-vars
+				} catch (error) {
+					// ignored
+				}
+			});
+			// eslint-disable-next-line no-unused-vars
+		} catch (error) {
+			// ignored
+		}
+	}
+	return urls;
+}
+
 function getSize(win, imageElement, computedStyle) {
 	let pxWidth = imageElement.naturalWidth;
 	let pxHeight = imageElement.naturalHeight;

package/core/index.js

@@ -735,6 +735,13 @@ class Processor {
 							if (imageData.currentSrc) {
 								imgElement.dataset.singleFileOriginURL = imgElement.getAttribute("src");
 								imgElement.setAttribute("src", imageData.currentSrc);
+								if (imageData.sourceSize && this.options.removeAlternativeImages) {
+									["width", "height"].forEach(attributeName => {
+										if (imageData.sourceSize[attributeName] != null) {
+											imgElement.setAttribute(attributeName, imageData.sourceSize[attributeName]);
+										}
+									});
+								}
 							}
 							if (this.options.loadDeferredContent) {
 								if ((!imgElement.getAttribute("src") || imgElement.getAttribute("src") == util.EMPTY_RESOURCE) && imgElement.getAttribute("data-src")) {

package/core/lib/processor-helper-common.js

@@ -230,7 +230,7 @@ class ProcessorHelperCommon {
 			if (options.saveOriginalURLs && !isDataURL(originSrcset)) {
 				resourceElement.setAttribute("data-sf-original-srcset", originSrcset);
 			}
-			if (!options.blockImages && !options.blockAlternativeImages) {
+			if (!options.blockImages && !options.removeAlternativeImages) {
 				const srcsetValues = await Promise.all(srcset.map(async srcsetValue => {
 					let resourceURL = normalizeURL(srcsetValue.url);
 					if (!testIgnoredPath(resourceURL)) {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-core",
-	"version": "1.6.27",
+	"version": "1.6.29",
 	"description": "SingleFile Core",
 	"author": "Gildas Lormeau",
 	"license": "AGPL-3.0-or-later",

package/processors/frame-tree/content/content-frame-tree.js

@@ -52,6 +52,24 @@ const INIT_REQUEST_MESSAGE = "singlefile.frameTree.initRequest";
 const ACK_INIT_REQUEST_MESSAGE = "singlefile.frameTree.ackInitRequest";
 const CLEANUP_REQUEST_MESSAGE = "singlefile.frameTree.cleanupRequest";
 const INIT_RESPONSE_MESSAGE = "singlefile.frameTree.initResponse";
+const GET_OPTIONS_MESSAGE = "singlefile.frameTree.getOptions";
+const FRAME_OPTION_NAMES = [
+	"removeHiddenElements",
+	"removeUnusedFonts",
+	"compressHTML",
+	"autoSaveExternalSave",
+	"moveStylesInHead",
+	"blockVideos",
+	"customStylesheet",
+	"manifest",
+	"loadDeferredContent",
+	"loadDeferredContentMaxIdleTime",
+	"loadDeferredContentKeepZoomLevel",
+	"loadDeferredContentMinZoomFactor",
+	"loadDeferredContentDispatchScrollEvent",
+	"loadDeferredContentBlockCookies",
+	"loadDeferredContentBlockStorage"
+];
 const TARGET_ORIGIN = "*";
 const TIMEOUT_INIT_REQUEST_MESSAGE = 5000;
 const TIMEOUT_INIT_RESPONSE_MESSAGE = 10000;
@@ -60,6 +78,7 @@ const WINDOW_ID_SEPARATOR = ".";
 const TOP_WINDOW = globalThis.window == globalThis.top;
 
 const browser = globalThis.browser;
+const RUNTIME_MESSAGING = Boolean(browser && browser.runtime && browser.runtime.sendMessage);
 const top = globalThis.top;
 const MessageChannel = globalThis.MessageChannel;
 const document = globalThis.document;
@@ -83,6 +102,9 @@ if (TOP_WINDOW) {
 				clearFrameTimeout("requestTimeouts", message.sessionId, message.windowId);
 				createFrameResponseTimeout(message.sessionId, message.windowId);
 				return Promise.resolve({});
+			} else if (message.method == GET_OPTIONS_MESSAGE) {
+				const session = sessions.get(message.sessionId);
+				return Promise.resolve({ options: session && session.options });
 			}
 		});
 	}
@@ -114,10 +136,15 @@ async function onMessage(event) {
 			}
 			if (!TOP_WINDOW) {
 				globalThis.stop();
-				if (message.options.loadDeferredContent) {
-					lazy.process(message.options);
+				if (!message.options && RUNTIME_MESSAGING) {
+					message.options = await getOptions(message.sessionId);
+				}
+				if (message.options) {
+					if (message.options.loadDeferredContent) {
+						lazy.process(message.options);
+					}
+					await initRequestAsync(message);
 				}
-				await initRequestAsync(message);
 			}
 		} else if (message.method == ACK_INIT_REQUEST_MESSAGE) {
 			clearFrameTimeout("requestTimeouts", message.sessionId, message.windowId);
@@ -133,12 +160,13 @@ async function onMessage(event) {
 
 function getAsync(options) {
 	const sessionId = getNewSessionId();
-	options = JSON.parse(JSON.stringify(options));
+	options = getFrameOptions(options);
 	return new Promise(resolve => {
 		sessions.set(sessionId, {
 			frames: [],
 			requestTimeouts: {},
 			responseTimeouts: {},
+			options,
 			resolve: frames => {
 				frames.sessionId = sessionId;
 				resolve(frames);
@@ -150,11 +178,12 @@ function getAsync(options) {
 
 function getSync(options) {
 	const sessionId = getNewSessionId();
-	options = JSON.parse(JSON.stringify(options));
+	options = getFrameOptions(options);
 	sessions.set(sessionId, {
 		frames: [],
 		requestTimeouts: {},
-		responseTimeouts: {}
+		responseTimeouts: {},
+		options
 	});
 	initRequestSync({ windowId, sessionId, options });
 	const frames = sessions.get(sessionId).frames;
@@ -171,6 +200,20 @@ function getNewSessionId() {
 	return globalThis.crypto.getRandomValues(new Uint32Array(32)).join("");
 }
 
+function getFrameOptions(options) {
+	return JSON.parse(JSON.stringify(options, FRAME_OPTION_NAMES));
+}
+
+async function getOptions(sessionId) {
+	try {
+		const response = await browser.runtime.sendMessage({ method: GET_OPTIONS_MESSAGE, sessionId });
+		return response && response.options;
+		// eslint-disable-next-line no-unused-vars
+	} catch (error) {
+		// ignored
+	}
+}
+
 function initRequestSync(message) {
 	const sessionId = message.sessionId;
 	delete globalThis._singleFile_cleaningUp;
@@ -282,7 +325,7 @@ function processFramesAsync(doc, frameElements, options, parentWindowId, session
 	frameElements.forEach((frameElement, frameIndex) => {
 		const windowId = parentWindowId + WINDOW_ID_SEPARATOR + frameIndex;
 		try {
-			sendMessage(frameElement.contentWindow, { method: INIT_REQUEST_MESSAGE, windowId, sessionId, options, scrolling: frameElement.scrolling, waitForFrames });
+			sendMessage(frameElement.contentWindow, { method: INIT_REQUEST_MESSAGE, windowId, sessionId, options: RUNTIME_MESSAGING ? undefined : options, scrolling: frameElement.scrolling, waitForFrames });
 			// eslint-disable-next-line no-unused-vars
 		} catch (error) {
 			// ignored

package/processors/hooks/content/content-hooks-frames-web.js

@@ -45,6 +45,7 @@
 	const GET_ADOPTED_STYLESHEETS_REQUEST_EVENT = "single-file-request-get-adopted-stylesheets";
 	const UNREGISTER_GET_ADOPTED_STYLESHEETS_REQUEST_EVENT = "single-file-unregister-request-get-adopted-stylesheets";
 	const GET_ADOPTED_STYLESHEETS_RESPONSE_EVENT = "single-file-response-get-adopted-stylesheets";
+	const CLEAR_ADOPTED_STYLESHEETS_CACHE_EVENT = "single-file-clear-adopted-stylesheets-cache";
 	const NEW_FONT_FACE_EVENT = "single-file-new-font-face";
 	const DELETE_FONT_EVENT = "single-file-delete-font";
 	const CLEAR_FONTS_EVENT = "single-file-clear-fonts";
@@ -97,6 +98,7 @@
 
 	let dispatchScrollEvent;
 	let adoptedStylesheetsData = new WeakMap();
+	let adoptedStylesheetsTexts = new WeakMap();
 	const shadowRootsData = new WeakMap();
 
 	init();
@@ -118,6 +120,7 @@
 		document.addEventListener(FETCH_SUPPORTED_REQUEST_EVENT, onFetchSupportedRequest);
 		document.addEventListener(FETCH_REQUEST_EVENT, onFetchRequest);
 		document.addEventListener(GET_ADOPTED_STYLESHEETS_REQUEST_EVENT, getAdoptedStylesheetsListener);
+		document.addEventListener(CLEAR_ADOPTED_STYLESHEETS_CACHE_EVENT, clearAdoptedStylesheetsCache);
 		document.addEventListener(BOOTSTRAP_EVENT, onBootstrap);
 	}
 
@@ -553,6 +556,7 @@
 		try {
 			const result = originalInsertRule.apply(this, [rule, arguments[1]]);
 			adoptedStylesheetsData.delete(this);
+			adoptedStylesheetsTexts.delete(this);
 			return result;
 		} catch (error) {
 			error.stack = error.message + "\n" + "    \n" + error.stack.trim().split("\n").slice(-1).join("\n");
@@ -566,6 +570,7 @@
 		try {
 			const result = originalDeleteRule.apply(this, [index]);
 			adoptedStylesheetsData.delete(this);
+			adoptedStylesheetsTexts.delete(this);
 			return result;
 		} catch (error) {
 			error.stack = error.message + "\n" + "    \n" + error.stack.trim().split("\n").slice(-1).join("\n");
@@ -600,9 +605,11 @@
 			const adoptedStyleSheets = Array.from(shadowRoot.adoptedStyleSheets).map(stylesheet => {
 				if (adoptedStylesheetsData.has(stylesheet)) {
 					return adoptedStylesheetsData.get(stylesheet);
+				} else if (adoptedStylesheetsTexts.has(stylesheet)) {
+					return adoptedStylesheetsTexts.get(stylesheet);
 				} else {
 					const text = Array.from(stylesheet.cssRules).map(cssRule => cssRule.cssText).join("\n");
-					adoptedStylesheetsData.set(stylesheet, text);
+					adoptedStylesheetsTexts.set(stylesheet, text);
 					return text;
 				}
 			});
@@ -620,6 +627,10 @@
 		event.currentTarget.removeEventListener(GET_ADOPTED_STYLESHEETS_REQUEST_EVENT, getAdoptedStylesheetsListener, { capture: true });
 	}
 
+	function clearAdoptedStylesheetsCache() {
+		adoptedStylesheetsTexts = new WeakMap();
+	}
+
 	async function getDetailObject(fontFace, src) {
 		const detail = {};
 		Object.keys(FONT_STYLE_PROPERTIES).forEach(descriptor => {