single-file-core 1.6.5 → 1.6.7

package/core/helper.js

@@ -67,7 +67,12 @@ const INVALID_ELEMENT_ATTRIBUTE_NAME = "data-" + SINGLE_FILE_PREFIX + "invalid-e
 const ASYNC_SCRIPT_ATTRIBUTE_NAME = "data-" + SINGLE_FILE_PREFIX + "async-script";
 const FLOW_ELEMENTS_SELECTOR = "*:not(base):not(link):not(meta):not(noscript):not(script):not(style):not(template):not(title)";
 const KEPT_TAG_NAMES = ["NOSCRIPT", "DISABLED-NOSCRIPT", "META", "LINK", "STYLE", "TITLE", "TEMPLATE", "SOURCE", "OBJECT", "SCRIPT", "HEAD", "BODY"];
+const INVOKABLE_ELEMENTS_SELECTOR = "[popover][id], dialog[id]";
+const INVOKER_ATTRIBUTE_NAMES = ["popovertarget", "commandfor"];
+const INVOKERS_SELECTOR = INVOKER_ATTRIBUTE_NAMES.map(attributeName => "[" + attributeName + "]").join(",");
 const IGNORED_TAG_NAMES = ["SCRIPT", "NOSCRIPT", "META", "LINK", "TEMPLATE"];
+const ROOT_PSEUDO_ELEMENT_NAMES = [":before", ":after"];
+const BLOCK_CONTAINER_DISPLAY_VALUES = ["block", "flow-root", "list-item", "inline-block", "table-cell", "table-caption"];
 const REGEXP_SIMPLE_QUOTES_STRING = /^'(.*?)'$/;
 const REGEXP_DOUBLE_QUOTES_STRING = /^"(.*?)"$/;
 const FONT_WEIGHTS = {
@@ -78,6 +83,11 @@ const FONT_WEIGHTS = {
 	lighter: "100"
 };
 const COMMENT_HEADER_LEGACY = "Archive processed by SingleFile";
+const ELEMENT_NODE_TYPE = 1;
+const TEXT_NODE_TYPE = 3;
+const REGEXP_QUOTED_STRING = /"((?:[^"\\]|\\.)*)"|'((?:[^'\\]|\\.)*)'/g;
+const REGEXP_GENERATED_CONTENT_IMAGE = /(url|image-set|-webkit-image-set|linear-gradient|radial-gradient|conic-gradient)\([^)]*\)/g;
+const REGEXP_BACKSLASH = /\\/;
 const SINGLE_FILE_UI_ELEMENT_CLASS = "single-file-ui-element";
 const INFOBAR_TAGNAME = infobar.INFOBAR_TAGNAME;
 const EMPTY_RESOURCE = "data:,";
@@ -164,10 +174,6 @@ export {
 	getPosterDataURI
 };
 
-// a poster is a still of a video the reader cannot play, so it is stored in a
-// lossy format: the same frame costs an order of magnitude less than as a PNG.
-// toDataURL falls back to PNG without reporting it when it does not support the
-// format, so the type it returned is tested instead of assumed
 function normalizeOptions(options) {
 	let normalizedOptions = options;
 	if (options) {
@@ -185,13 +191,18 @@ function normalizeOptions(options) {
 }
 
 function getPosterDataURI(canvasElement) {
-	for (const contentType of POSTER_CONTENT_TYPES) {
-		const dataURI = canvasElement.toDataURL(contentType, POSTER_QUALITY);
-		if (dataURI.startsWith("data:" + contentType)) {
+	if (canvasElement.width && canvasElement.height) {
+		for (const contentType of POSTER_CONTENT_TYPES) {
+			const dataURI = canvasElement.toDataURL(contentType, POSTER_QUALITY);
+			if (dataURI.startsWith("data:" + contentType)) {
+				return dataURI;
+			}
+		}
+		const dataURI = canvasElement.toDataURL("image/png");
+		if (dataURI != EMPTY_RESOURCE) {
 			return dataURI;
 		}
 	}
-	return canvasElement.toDataURL("image/png");
 }
 
 let userScriptHandlerObserver;
@@ -276,6 +287,9 @@ function preProcessDoc(doc, win, options) {
 	if (win && doc.documentElement) {
 		markInvalidNesting(doc);
 		elementsInfo = getElementsInfo(win, doc, doc.documentElement, options);
+		if (options.removeUnusedFonts && doc.defaultView) {
+			getRootElementUsedFonts(win, doc.documentElement, elementsInfo);
+		}
 		if (options.moveStylesInHead) {
 			doc.querySelectorAll("body style, body ~ style").forEach(element => {
 				const computedStyle = getComputedStyle(win, element);
@@ -292,6 +306,7 @@ function preProcessDoc(doc, win, options) {
 			posters: [],
 			videos: [],
 			usedFonts: [],
+			usedFontsCharacters: new Map(),
 			shadowRoots: [],
 			markedElements: []
 		};
@@ -314,6 +329,7 @@ function preProcessDoc(doc, win, options) {
 		posters: elementsInfo.posters,
 		videos: elementsInfo.videos,
 		usedFonts: Array.from(elementsInfo.usedFonts.values()),
+		usedFontsCharacters: serializeUsedFontsCharacters(elementsInfo.usedFontsCharacters),
 		shadowRoots: elementsInfo.shadowRoots,
 		referrer,
 		markedElements: elementsInfo.markedElements,
@@ -414,16 +430,19 @@ function fixInvalidNesting(document, NESTING_TRACK_ID_ATTRIBUTE_NAME, preventCle
 	}
 }
 
-function getElementsInfo(win, doc, element, options, data = { usedFonts: new Map(), canvases: [], images: [], posters: [], videos: [], shadowRoots: [], markedElements: [] }, adoptedStyleSheetsCache = new Map(), ascendantHidden) {
+function getElementsInfo(win, doc, element, options, data = { usedFonts: new Map(), usedFontsCharacters: new Map(), canvases: [], images: [], posters: [], videos: [], shadowRoots: [], markedElements: [] }, adoptedStyleSheetsCache = new Map(), ascendantHidden) {
 	if (element.childNodes) {
 		const elements = Array.from(element.childNodes).filter(node => (node instanceof win.HTMLElement) || (node instanceof win.SVGElement) || (node instanceof globalThis.HTMLElement) || (node instanceof globalThis.SVGElement));
 		elements.forEach(element => {
-			let elementHidden, elementKept, computedStyle;
+			let elementHidden, elementKept, computedStyle, headChild;
 			if (!options.autoSaveExternalSave && (options.removeHiddenElements || options.removeUnusedFonts || options.compressHTML)) {
 				computedStyle = getComputedStyle(win, element);
+				if (options.removeHiddenElements || options.removeUnusedFonts) {
+					headChild = Boolean(element.closest("html > head"));
+				}
 				if ((element instanceof win.HTMLElement) || (element instanceof globalThis.HTMLElement)) {
 					if (options.removeHiddenElements) {
-						elementKept = ((ascendantHidden || element.closest("html > head")) && KEPT_TAG_NAMES.includes(element.tagName.toUpperCase())) || element.closest("details");
+						elementKept = ((ascendantHidden || headChild) && KEPT_TAG_NAMES.includes(element.tagName.toUpperCase())) || element.closest("details") || testInvokedElement(element, doc, data);
 						if (!elementKept) {
 							elementHidden = ascendantHidden || testHiddenElement(element, computedStyle);
 							if (elementHidden && !IGNORED_TAG_NAMES.includes(element.tagName.toUpperCase())) {
@@ -441,11 +460,28 @@ function getElementsInfo(win, doc, element, options, data = { usedFonts: new Map
 							data.markedElements.push(element);
 						}
 					}
-					if (options.removeUnusedFonts && doc.defaultView) {
-						getUsedFont(computedStyle, data.usedFonts);
-						getUsedFont(getComputedStyle(win, element, ":first-letter"), data.usedFonts);
-						getUsedFont(getComputedStyle(win, element, ":before"), data.usedFonts);
-						getUsedFont(getComputedStyle(win, element, ":after"), data.usedFonts);
+					if (options.removeUnusedFonts && doc.defaultView && !headChild) {
+						const elementCharacters = getElementCharacters(win, element);
+						getUsedFont(computedStyle, data.usedFonts, data.usedFontsCharacters, elementCharacters);
+						getUsedFont(getComputedStyle(win, element, ":first-letter"), data.usedFonts, data.usedFontsCharacters, elementCharacters);
+						const beforeStyle = getComputedStyle(win, element, ":before");
+						getUsedFont(beforeStyle, data.usedFonts, data.usedFontsCharacters, getPseudoElementCharacters(beforeStyle));
+						const afterStyle = getComputedStyle(win, element, ":after");
+						getUsedFont(afterStyle, data.usedFonts, data.usedFontsCharacters, getPseudoElementCharacters(afterStyle));
+						const display = computedStyle ? computedStyle.getPropertyValue("display") : "";
+						const tagName = element.tagName.toUpperCase();
+						if (display == "list-item") {
+							getUsedFont(getComputedStyle(win, element, "::marker"), data.usedFonts);
+						}
+						if (BLOCK_CONTAINER_DISPLAY_VALUES.includes(display)) {
+							getUsedFont(getComputedStyle(win, element, "::first-line"), data.usedFonts, data.usedFontsCharacters, elementCharacters);
+						}
+						if ((tagName == "INPUT" || tagName == "TEXTAREA") && element.getAttribute("placeholder")) {
+							getUsedFont(getComputedStyle(win, element, "::placeholder"), data.usedFonts, data.usedFontsCharacters, { characters: element.getAttribute("placeholder") });
+						}
+						if (tagName == "INPUT" && element.type == "file") {
+							getUsedFont(getComputedStyle(win, element, "::file-selector-button"), data.usedFonts);
+						}
 					}
 				}
 			}
@@ -458,15 +494,15 @@ function getElementsInfo(win, doc, element, options, data = { usedFonts: new Map
 				data.shadowRoots.push(shadowRootInfo);
 				try {
 					if (shadowRoot.adoptedStyleSheets) {
-						
-							const listener = event => shadowRootInfo.adoptedStyleSheets = event.detail.adoptedStyleSheets;
-							shadowRoot.addEventListener(GET_ADOPTED_STYLESHEETS_RESPONSE_EVENT, listener);
-							shadowRoot.dispatchEvent(new CustomEvent(GET_ADOPTED_STYLESHEETS_REQUEST_EVENT, { bubbles: true }));
-							if (!shadowRootInfo.adoptedStyleSheets) {
-								element.dispatchEvent(new CustomEvent(GET_ADOPTED_STYLESHEETS_REQUEST_EVENT, { bubbles: true }));
-							}
-							shadowRoot.removeEventListener(GET_ADOPTED_STYLESHEETS_RESPONSE_EVENT, listener);
-						
+
+						const listener = event => shadowRootInfo.adoptedStyleSheets = event.detail.adoptedStyleSheets;
+						shadowRoot.addEventListener(GET_ADOPTED_STYLESHEETS_RESPONSE_EVENT, listener);
+						shadowRoot.dispatchEvent(new CustomEvent(GET_ADOPTED_STYLESHEETS_REQUEST_EVENT, { bubbles: true }));
+						if (!shadowRootInfo.adoptedStyleSheets) {
+							element.dispatchEvent(new CustomEvent(GET_ADOPTED_STYLESHEETS_REQUEST_EVENT, { bubbles: true }));
+						}
+						shadowRoot.removeEventListener(GET_ADOPTED_STYLESHEETS_RESPONSE_EVENT, listener);
+
 					}
 					// eslint-disable-next-line no-unused-vars
 				} catch (error) {
@@ -527,8 +563,6 @@ function getStylesheetsContent(styleSheets, adoptedStyleSheetsCache = new Map())
 	}
 }
 
-// an untouched canvas is fully transparent, so it encodes exactly like a blank one of the same
-// size: comparing the two is cheaper than reading the pixels back and needs no drawing context
 function isBlankCanvas(doc, element, dataURI) {
 	const blankElement = doc.createElement("canvas");
 	blankElement.width = element.width;
@@ -545,9 +579,6 @@ function getResourcesInfo(win, doc, element, options, data, elementHidden, compu
 		};
 		try {
 			const dataURI = element.toDataURL("image/png");
-			// a canvas in a page SingleFile has already saved is empty, no script ran to draw into
-			// it, and the picture it displays is the background image the previous save left
-			// behind. An empty bitmap must not overwrite it
 			const backgroundImage = canvasComputedStyle ? canvasComputedStyle.getPropertyValue("background-image") : element.style.getPropertyValue("background-image");
 			if (backgroundImage && backgroundImage != "none" && isBlankCanvas(doc, element, dataURI)) {
 				canvasData.blank = true;
@@ -605,8 +636,6 @@ function getResourcesInfo(win, doc, element, options, data, elementHidden, compu
 			});
 			element.setAttribute(VIDEO_ATTRIBUTE_NAME, data.videos.length - 1);
 		}
-		// the posters are only inserted when the videos are blocked, capturing them
-		// otherwise encodes a frame per video for nothing
 		if (options.blockVideos && !element.getAttribute("poster")) {
 			const canvasElement = doc.createElement("canvas");
 			const context = canvasElement.getContext("2d");
@@ -614,9 +643,12 @@ function getResourcesInfo(win, doc, element, options, data, elementHidden, compu
 			canvasElement.height = element.videoHeight;
 			try {
 				context.drawImage(element, 0, 0, canvasElement.width, canvasElement.height);
-				data.posters.push(getPosterDataURI(canvasElement));
-				element.setAttribute(POSTER_ATTRIBUTE_NAME, data.posters.length - 1);
-				data.markedElements.push(element);
+				const posterDataURI = getPosterDataURI(canvasElement);
+				if (posterDataURI) {
+					data.posters.push(posterDataURI);
+					element.setAttribute(POSTER_ATTRIBUTE_NAME, data.posters.length - 1);
+					data.markedElements.push(element);
+				}
 				// eslint-disable-next-line no-unused-vars
 			} catch (error) {
 				// ignored
@@ -660,7 +692,7 @@ function getResourcesInfo(win, doc, element, options, data, elementHidden, compu
 	}
 }
 
-function getUsedFont(computedStyle, usedFonts) {
+function getUsedFont(computedStyle, usedFonts, usedFontsCharacters, drawnCharacters) {
 	if (computedStyle) {
 		const fontStyle = computedStyle.getPropertyValue("font-style") || "normal";
 		computedStyle.getPropertyValue("font-family").split(",").forEach(fontFamilyName => {
@@ -670,9 +702,97 @@ function getUsedFont(computedStyle, usedFonts) {
 				const fontVariant = computedStyle.getPropertyValue("font-variant") || "normal";
 				const value = [fontFamilyName, fontWeight, fontStyle, fontVariant];
 				usedFonts.set(JSON.stringify(value), [fontFamilyName, fontWeight, fontStyle, fontVariant]);
+				if (usedFontsCharacters && drawnCharacters) {
+					addUsedFontCharacters(usedFontsCharacters, fontFamilyName, fontStyle, drawnCharacters);
+				}
+			}
+		});
+	}
+}
+
+function addUsedFontCharacters(usedFontsCharacters, fontFamilyName, fontStyle, drawnCharacters) {
+	const key = fontFamilyName + "|" + fontStyle;
+	let bucket = usedFontsCharacters.get(key);
+	if (!bucket) {
+		bucket = { fontFamily: fontFamilyName, fontStyle, charCodes: new Set(), unknown: false };
+		usedFontsCharacters.set(key, bucket);
+	}
+	if (drawnCharacters.unknown) {
+		bucket.unknown = true;
+	}
+	for (const character of drawnCharacters.characters) {
+		bucket.charCodes.add(character.codePointAt(0));
+	}
+}
+
+function getElementCharacters(win, element) {
+	let characters = "";
+	const tagName = element.tagName && element.tagName.toUpperCase();
+	if (tagName == "INPUT" || tagName == "TEXTAREA" || tagName == "BUTTON") {
+		characters += (element.value || "") + (element.getAttribute("placeholder") || "");
+	}
+	if (element.childNodes) {
+		Array.from(element.childNodes).forEach(node => {
+			if (node.nodeType == TEXT_NODE_TYPE) {
+				characters += node.data;
+			} else if (node.nodeType == ELEMENT_NODE_TYPE &&
+				!((node instanceof win.HTMLElement) || (node instanceof win.SVGElement) ||
+					(node instanceof globalThis.HTMLElement) || (node instanceof globalThis.SVGElement))) {
+				characters += node.textContent;
 			}
 		});
 	}
+	return { characters };
+}
+
+// getElementsInfo iterates the children of the element it is given, so the root is never visited.
+// An inherited property costs nothing there, since every descendant carries the root's value, but a
+// pseudo-element on the root has no descendant to speak for it: html::before renders a box with a
+// font of its own and nothing else records it. Only a pseudo that generates content is recorded,
+// because the computed style of a pseudo that draws nothing still reports the inherited family and
+// registering that would mark the whole root font stack as used.
+function getRootElementUsedFonts(win, element, data) {
+	ROOT_PSEUDO_ELEMENT_NAMES.forEach(pseudoElementName => {
+		const computedStyle = getComputedStyle(win, element, pseudoElementName);
+		const drawnCharacters = getPseudoElementCharacters(computedStyle);
+		if (drawnCharacters.characters || drawnCharacters.unknown) {
+			getUsedFont(computedStyle, data.usedFonts, data.usedFontsCharacters, drawnCharacters);
+		}
+	});
+}
+
+function getPseudoElementCharacters(computedStyle) {
+	const content = computedStyle && computedStyle.getPropertyValue("content");
+	if (!content || content == "none" || content == "normal") {
+		return { characters: "" };
+	}
+	let characters = "";
+	REGEXP_QUOTED_STRING.lastIndex = 0;
+	let match = REGEXP_QUOTED_STRING.exec(content);
+	while (match) {
+		characters += match[1] === undefined ? match[2] : match[1];
+		match = REGEXP_QUOTED_STRING.exec(content);
+	}
+	const remainder = content
+		.replace(REGEXP_QUOTED_STRING, "")
+		.replace(REGEXP_GENERATED_CONTENT_IMAGE, "")
+		.trim();
+	return { characters, unknown: Boolean(remainder) || REGEXP_BACKSLASH.test(characters) };
+}
+
+function serializeUsedFontsCharacters(usedFontsCharacters) {
+	return Array.from(usedFontsCharacters.values()).map(bucket => {
+		const ranges = [];
+		Array.from(bucket.charCodes).sort((charCode1, charCode2) => charCode1 - charCode2).forEach(charCode => {
+			const lastRange = ranges[ranges.length - 1];
+			if (lastRange && charCode == lastRange[1] + 1) {
+				lastRange[1] = charCode;
+			} else {
+				ranges.push([charCode, charCode]);
+			}
+		});
+		return [bucket.fontFamily, bucket.fontStyle, ranges, bucket.unknown ? 1 : 0];
+	});
 }
 
 function getShadowRoot(element) {
@@ -699,13 +819,30 @@ function normalizeFontFamily(fontFamilyName = "") {
 	return removeQuotes(cssUnescape.process(fontFamilyName.trim())).toLowerCase();
 }
 
+function testInvokedElement(element, doc, data) {
+	const invokableElement = element.closest(INVOKABLE_ELEMENTS_SELECTOR);
+	if (invokableElement) {
+		if (!data.invokedIds) {
+			data.invokedIds = new Set();
+			doc.querySelectorAll(INVOKERS_SELECTOR).forEach(invoker => INVOKER_ATTRIBUTE_NAMES.forEach(attributeName => {
+				if (invoker.hasAttribute(attributeName)) {
+					data.invokedIds.add(invoker.getAttribute(attributeName));
+				}
+			}));
+		}
+		return data.invokedIds.has(invokableElement.id);
+	}
+	return false;
+}
+
 function testHiddenElement(element, computedStyle) {
 	let hidden = false;
 	if (computedStyle) {
 		const display = computedStyle.getPropertyValue("display");
 		const opacity = computedStyle.getPropertyValue("opacity");
 		const visibility = computedStyle.getPropertyValue("visibility");
-		hidden = display == "none";
+		const tagName = element.tagName && element.tagName.toUpperCase();
+		hidden = display == "none" || (visibility == "hidden" && tagName == "IFRAME");
 		if (!hidden && (opacity == "0" || visibility == "hidden") && element.getBoundingClientRect) {
 			const boundingRect = element.getBoundingClientRect();
 			hidden = !boundingRect.width && !boundingRect.height;

package/core/index.js

@@ -145,7 +145,9 @@ const STAGES = [{
 	]
 }, {
 	sequential: [
-		{ option: "removeAlternativeImages", action: "removeAlternativeImages" }
+		{ option: "removeAlternativeImages", action: "removeAlternativeImages" },
+		{ action: "groupDuplicateFonts" },
+		{ action: "groupDuplicateImages" }
 	],
 	parallel: [
 		{ option: "removeAlternativeFonts", action: "removeAlternativeFonts" },
@@ -199,6 +201,7 @@ class Runner {
 			this.options.posters = docData.posters;
 			this.options.videos = docData.videos;
 			this.options.usedFonts = docData.usedFonts;
+			this.options.usedFontsCharacters = docData.usedFontsCharacters;
 			this.options.shadowRoots = docData.shadowRoots;
 			this.options.referrer = docData.referrer;
 			this.options.adoptedStyleSheets = docData.adoptedStyleSheets;
@@ -222,6 +225,7 @@ class Runner {
 		await this.onprogress(new ProgressEvent(RESOURCES_INITIALIZING, { pageURL: this.options.url, options: this.options }));
 		await this.executeStage(RESOLVE_URLS_STAGE);
 		this.pendingPromises = this.executeStage(REPLACE_DATA_STAGE);
+		this.pendingPromises.catch(() => { });
 		if (this.root && this.options.doc) {
 			util.postProcessDoc(this.options.doc, this.markedElements, this.invalidElements);
 		}
@@ -282,7 +286,7 @@ class Runner {
 			log("**** STARTED STAGE", step, "****");
 		}
 		const frame = !this.root;
-		await this.onprogress(new ProgressEvent(STAGE_STARTED, { pageURL: this.options.url, step, frame, options: this.options }));
+		const stageStartedPromise = this.onprogress(new ProgressEvent(STAGE_STARTED, { pageURL: this.options.url, step, frame, options: this.options }));
 		for (const task of STAGES[step].sequential) {
 			let startTime;
 			if (DEBUG) {
@@ -314,6 +318,7 @@ class Runner {
 		} else {
 			parallelTasksPromise = Promise.resolve();
 		}
+		await stageStartedPromise;
 		await this.onprogress(new ProgressEvent(STAGE_ENDED, { pageURL: this.options.url, step, frame, options: this.options }));
 		if (DEBUG) {
 			log("**** ENDED   STAGE", step, "****");
@@ -341,19 +346,25 @@ class BatchRequest {
 		return new Promise((resolve, reject) => {
 			const requestKey = JSON.stringify([resourceURL, asBinary, expectedType, baseURI, blockMixedContent, contentType]);
 			let resourceRequests = this.requests.get(requestKey);
-			if (!resourceRequests) {
-				resourceRequests = [];
-				this.requests.set(requestKey, resourceRequests);
-			}
-			const callbacks = { resolve, reject };
-			resourceRequests.push(callbacks);
-			if (groupDuplicates) {
-				let duplicateRequests = this.duplicates.get(requestKey);
-				if (!duplicateRequests) {
-					duplicateRequests = [];
-					this.duplicates.set(requestKey, duplicateRequests);
+			if (this.cancelled) {
+				reject();
+			} else if (this.running && !resourceRequests) {
+				reject(new Error("Resource requested after the batch started: " + resourceURL));
+			} else {
+				if (!resourceRequests) {
+					resourceRequests = [];
+					this.requests.set(requestKey, resourceRequests);
+				}
+				const callbacks = { resolve, reject };
+				resourceRequests.push(callbacks);
+				if (groupDuplicates) {
+					let duplicateRequests = this.duplicates.get(requestKey);
+					if (!duplicateRequests) {
+						duplicateRequests = [];
+						this.duplicates.set(requestKey, duplicateRequests);
+					}
+					duplicateRequests.push(callbacks);
 				}
-				duplicateRequests.push(callbacks);
 			}
 		});
 	}
@@ -363,6 +374,7 @@ class BatchRequest {
 	}
 
 	run(onloadListener, options) {
+		this.running = true;
 		const resourceURLs = [...this.requests.keys()];
 		let indexResource = 0;
 		return Promise.all(resourceURLs.map(async requestKey => {
@@ -385,7 +397,6 @@ class BatchRequest {
 					acceptHeaders: options.acceptHeaders,
 					networkTimeout: options.networkTimeout
 				});
-				await onloadListener({ url: resourceURL });
 				if (!this.cancelled) {
 					const extension = util.getContentTypeExtension(content.contentType) || util.getFilenameExtension(resourceURL, options.filenameReplacedCharacters, options.filenameReplacementCharacter, options.filenameReplacementCharacters);
 					resourceRequests.forEach(callbacks => {
@@ -395,11 +406,10 @@ class BatchRequest {
 					});
 				}
 			} catch (error) {
-				indexResource = indexResource + 1;
-				await onloadListener({ url: resourceURL });
 				resourceRequests.forEach(resourceRequest => resourceRequest.reject(error));
 			}
 			this.requests.delete(requestKey);
+			await onloadListener({ url: resourceURL });
 		}));
 	}
 
@@ -427,6 +437,8 @@ const DISABLED_SCRIPT_URI = "javascript:void(0)";
 const SCRIPT_URI_ATTRIBUTE_NAMES = ["href", "src", "action", "formaction", "data"];
 const UTF8_CHARSET = "utf-8";
 const TAINTED_CANVAS_WARNING_MESSAGE = "SingleFile: canvas elements tainted by a cross-origin resource, dropped from the page:";
+const EMPTY_RESOURCE = "data:,";
+const ORDER_DEPENDENT_RULE = /@(layer|import)\b/i;
 
 class Processor {
 	constructor(options, processorHelper, batchRequest) {
@@ -457,7 +469,7 @@ class Processor {
 		}
 		this.maxResources = this.batchRequest.getMaxResources();
 		if (!this.options.removeFrames && this.options.frames) {
-			this.options.frames.forEach(frameData => this.maxResources += frameData.maxResources || 0);
+			this.options.frames.forEach(frameData => this.maxResources += frameData.runner ? frameData.runner.batchRequest.getMaxResources() : 0);
 		}
 		this.stats.set("processed", "resources", this.maxResources);
 	}
@@ -785,7 +797,7 @@ class Processor {
 				const attributeValue = element.getAttribute(util.POSTER_ATTRIBUTE_NAME);
 				if (attributeValue) {
 					const posterURL = this.options.posters[Number(attributeValue)];
-					if (!videoElement.getAttribute("poster") && posterURL) {
+					if (!videoElement.getAttribute("poster") && posterURL && posterURL != EMPTY_RESOURCE) {
 						videoElement.setAttribute("poster", posterURL);
 					}
 				}
@@ -1204,12 +1216,15 @@ class Processor {
 			} else {
 				videoElement = element.parentElement;
 			}
-			if (!videoElement.poster) {
+			if (!videoElement.poster || videoElement.poster == EMPTY_RESOURCE) {
 				const attributeValue = videoElement.getAttribute(util.VIDEO_ATTRIBUTE_NAME);
 				if (attributeValue) {
 					const videoData = this.options.videos[Number(attributeValue)];
 					const src = videoData.src || videoElement.src;
 					if (src) {
+						const blankPosterURI = "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='" +
+							(videoData.size.videoWidth || videoData.size.pxWidth) + "' height='" +
+							(videoData.size.videoHeight || videoData.size.pxHeight) + "'%3E%3C/svg%3E";
 						const temporaryVideoElement = this.doc.createElement("video");
 						temporaryVideoElement.src = src;
 						temporaryVideoElement.style.setProperty("width", videoData.size.pxWidth + "px", "important");
@@ -1222,21 +1237,21 @@ class Processor {
 						return new Promise(resolve => {
 							temporaryVideoElement.currentTime = videoData.currentTime;
 							temporaryVideoElement.oncanplay = () => {
-								canvasElement.width = videoData.size.videoWidth;
-								canvasElement.height = videoData.size.videoHeight;
+								canvasElement.width = temporaryVideoElement.videoWidth || videoData.size.videoWidth;
+								canvasElement.height = temporaryVideoElement.videoHeight || videoData.size.videoHeight;
 								context.drawImage(temporaryVideoElement, 0, 0, canvasElement.width, canvasElement.height);
 								try {
-									videoElement.poster = util.getPosterDataURI(canvasElement);
+									videoElement.poster = util.getPosterDataURI(canvasElement) || blankPosterURI;
 									// eslint-disable-next-line no-unused-vars
 								} catch (error) {
-									videoElement.poster = "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='" + videoData.size.videoWidth + "' height='" + videoData.size.videoHeight + "'%3E%3C/svg%3E";
+									videoElement.poster = blankPosterURI;
 									// ignored
 								}
 								temporaryVideoElement.remove();
 								resolve();
 							};
 							temporaryVideoElement.onerror = () => {
-								videoElement.poster = "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='" + videoData.size.videoWidth + "' height='" + videoData.size.videoHeight + "'%3E%3C/svg%3E";
+								videoElement.poster = blankPosterURI;
 								temporaryVideoElement.remove();
 								resolve();
 							};
@@ -1261,23 +1276,32 @@ class Processor {
 	}
 
 	async resolveStylesheetsURLs() {
-		const stylesContents = [];
+		const styleElementGroups = new Map();
 		this.options.inlineStylesheets = new Map();
 		this.options.inlineStylesheetsRefs = new Map();
 		this.doc.querySelectorAll("style").forEach(styleElement => {
-			if (styleElement.textContent) {
-				const indexContent = stylesContents.indexOf(styleElement.textContent);
-				if (indexContent == -1) {
-					this.options.inlineStylesheets.set(stylesContents.length, {
-						styleElement,
-						content: styleElement.textContent
-					});
-					stylesContents.push(styleElement.textContent);
-				} else {
-					this.options.inlineStylesheetsRefs.set(styleElement, indexContent);
+			const content = styleElement.textContent;
+			if (content) {
+				if (!styleElementGroups.has(content)) {
+					styleElementGroups.set(content, []);
 				}
+				styleElementGroups.get(content).push(styleElement);
 			}
 		});
+		let indexContent = 0;
+		styleElementGroups.forEach((styleElements, content) => {
+			const masterIndex = ORDER_DEPENDENT_RULE.test(content) ? 0 : styleElements.length - 1;
+			this.options.inlineStylesheets.set(indexContent, {
+				styleElement: styleElements[masterIndex],
+				content
+			});
+			styleElements.forEach((styleElement, index) => {
+				if (index != masterIndex) {
+					this.options.inlineStylesheetsRefs.set(styleElement, indexContent);
+				}
+			});
+			indexContent++;
+		});
 		await Promise.all(Array.from(this.doc.querySelectorAll("style, link[rel*=stylesheet]:not([disabled])")).map(async element => {
 			const options = Object.assign({}, this.options, { charset: this.charset });
 			let mediaText;
@@ -1346,19 +1370,20 @@ class Processor {
 					stylesheets: [],
 					url,
 					usedFonts: [],
+					usedFontsCharacters: [],
 					videos: [],
 					worklets: []
 				};
 				this.options.frames.push(frameData);
 				frameData.windowId = (this.options.windowId || "0") + "." + this.options.frames.length;
 				frameElement.setAttribute(util.WIN_ID_ATTRIBUTE_NAME, frameData.windowId);
-				await initializeProcessor(frameData, frameElement, null, this.batchRequest, Object.assign({}, this.options));
+				await initializeProcessor(frameData, frameElement, null, Object.assign({}, this.options));
 			} else {
 				const frameWindowId = frameElement.getAttribute(util.WIN_ID_ATTRIBUTE_NAME);
 				if (this.options.frames && frameWindowId) {
 					const frameData = this.options.frames.find(frame => frame.windowId == frameWindowId);
 					if (frameData && frameData.content) {
-						await initializeProcessor(frameData, frameElement, frameWindowId, this.batchRequest, Object.assign({}, this.options));
+						await initializeProcessor(frameData, frameElement, frameWindowId, Object.assign({}, this.options));
 					}
 				}
 			}
@@ -1369,7 +1394,7 @@ class Processor {
 			}
 		});
 
-		async function initializeProcessor(frameData, frameElement, frameWindowId, batchRequest, options) {
+		async function initializeProcessor(frameData, frameElement, frameWindowId, options) {
 			options.insertSingleFileComment = false;
 			options.insertCanonicalLink = false;
 			options.insertMetaNoIndex = false;
@@ -1390,6 +1415,7 @@ class Processor {
 			options.posters = frameData.posters;
 			options.videos = frameData.videos;
 			options.usedFonts = frameData.usedFonts;
+			options.usedFontsCharacters = frameData.usedFontsCharacters;
 			options.shadowRoots = frameData.shadowRoots;
 			options.scrollPosition = frameData.scrollPosition;
 			options.scrolling = frameData.scrolling;
@@ -1398,7 +1424,6 @@ class Processor {
 			frameData.frameElement = frameElement;
 			await frameData.runner.loadPage();
 			await frameData.runner.initialize();
-			frameData.maxResources = batchRequest.getMaxResources();
 		}
 	}
 
@@ -1538,6 +1563,14 @@ class Processor {
 		util.removeAlternativeImages(this.doc);
 	}
 
+	groupDuplicateFonts() {
+		this.processorHelper.groupDuplicateFonts(this.stylesheets, this.resources.fonts, this.options);
+	}
+
+	groupDuplicateImages() {
+		this.processorHelper.groupDuplicateImages(this.doc, this.stylesheets, this.styles, this.resources.images);
+	}
+
 	async removeAlternativeFonts() {
 		await this.processorHelper.removeAlternativeFonts(this.doc, this.stylesheets, this.resources.fonts, this.options.fontTests);
 	}
@@ -1545,7 +1578,7 @@ class Processor {
 	async processFrames() {
 		if (this.options.frames) {
 			const frameElements = Array.from(this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]"));
-			await Promise.all(frameElements.map(async frameElement => {
+			const capturedFrames = await Promise.all(frameElements.map(async frameElement => {
 				const frameWindowId = frameElement.getAttribute(util.WIN_ID_ATTRIBUTE_NAME);
 				if (frameWindowId) {
 					const frameData = this.options.frames.find(frame => frame.windowId == frameWindowId);
@@ -1556,8 +1589,7 @@ class Processor {
 							await frameData.runner.run();
 							const pageData = await frameData.runner.getPageData();
 							frameElement.removeAttribute(util.WIN_ID_ATTRIBUTE_NAME);
-							this.processorHelper.processFrame(frameElement, pageData, this.options, this.resources, frameWindowId, frameData);
-							this.stats.addAll(pageData);
+							return { frameElement, pageData, frameWindowId, frameData };
 						} else {
 							frameElement.removeAttribute(util.WIN_ID_ATTRIBUTE_NAME);
 							this.stats.add("discarded", "frames", 1);
@@ -1565,6 +1597,13 @@ class Processor {
 					}
 				}
 			}));
+			capturedFrames.forEach(capturedFrame => {
+				if (capturedFrame) {
+					const { frameElement, pageData, frameWindowId, frameData } = capturedFrame;
+					this.processorHelper.processFrame(frameElement, pageData, this.options, this.resources, frameWindowId, frameData);
+					this.stats.addAll(pageData);
+				}
+			});
 		}
 	}