single-file-core 1.6.25 → 1.6.27

package/core/helper.js

@@ -376,10 +376,65 @@ function preProcessDoc(doc, win, options) {
 		markedElements: elementsInfo.markedElements,
 		invalidElements,
 		scrollPosition: { x: win.scrollX, y: win.scrollY },
-		adoptedStyleSheets: getStylesheetsContent(doc.adoptedStyleSheets)
+		adoptedStyleSheets: getStylesheetsContent(doc.adoptedStyleSheets),
+		manifestData: options.manifest ? getManifestData(doc, win) : undefined
 	};
 }
 
+function getManifestData(doc, win) {
+	const manifestData = { url: doc.documentURI, timings: {} };
+	try {
+		const lastModified = new Date(doc.lastModified);
+		if (!isNaN(lastModified)) {
+			manifestData.lastModified = lastModified.toISOString();
+		}
+		// eslint-disable-next-line no-unused-vars
+	} catch (error) {
+		// ignored
+	}
+	try {
+		const performance = win.performance;
+		manifestData.timeOrigin = new Date(performance.timeOrigin).toISOString();
+		const [navigation] = performance.getEntriesByType("navigation");
+		if (navigation) {
+			manifestData.navigation = getTimingData(navigation);
+			manifestData.navigation.type = navigation.type;
+			manifestData.navigation.redirectCount = navigation.redirectCount;
+		}
+		performance.getEntriesByType("resource").forEach(entry => manifestData.timings[entry.name] = getTimingData(entry));
+		// eslint-disable-next-line no-unused-vars
+	} catch (error) {
+		// ignored
+	}
+	try {
+		manifestData.environment = {
+			userAgent: win.navigator.userAgent,
+			viewport: { width: win.innerWidth, height: win.innerHeight, devicePixelRatio: win.devicePixelRatio },
+			media: {
+				prefersColorScheme: win.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light",
+				prefersReducedMotion: win.matchMedia("(prefers-reduced-motion: reduce)").matches ? "reduce" : "no-preference"
+			}
+		};
+		// eslint-disable-next-line no-unused-vars
+	} catch (error) {
+		// ignored
+	}
+	return manifestData;
+}
+
+function getTimingData(entry) {
+	const timingData = {
+		transferSize: entry.transferSize,
+		encodedBodySize: entry.encodedBodySize,
+		decodedBodySize: entry.decodedBodySize,
+		nextHopProtocol: entry.nextHopProtocol
+	};
+	if (entry.responseStatus) {
+		timingData.status = entry.responseStatus;
+	}
+	return timingData;
+}
+
 function insertCustomStylesheet(doc, content) {
 	const parentElement = doc.body || doc.documentElement;
 	if (parentElement) {
@@ -1351,7 +1406,7 @@ async function getDataURI(blob) {
 
 async function digest(algo, text) {
 	try {
-		const data = new TextEncoder("utf-8").encode(text);
+		const data = typeof text == "string" ? new TextEncoder("utf-8").encode(text) : text instanceof Uint8Array ? text : new Uint8Array(text);
 		const hash = globalThis.crypto && crypto.subtle ? await crypto.subtle.digest(algo, data) : sha.digest(algo, data);
 		return hex(hash);
 		// eslint-disable-next-line no-unused-vars

package/core/index.js

@@ -207,6 +207,7 @@ class Runner {
 			this.options.shadowRoots = docData.shadowRoots;
 			this.options.referrer = docData.referrer;
 			this.options.adoptedStyleSheets = docData.adoptedStyleSheets;
+			this.options.manifestData = docData.manifestData;
 			this.markedElements = docData.markedElements;
 			this.invalidElements = docData.invalidElements;
 		}
@@ -403,8 +404,12 @@ class BatchRequest {
 					baseURI,
 					blockMixedContent,
 					acceptHeaders: options.acceptHeaders,
-					networkTimeout: options.networkTimeout
+					networkTimeout: options.networkTimeout,
+					manifestEntries: options.manifestEntries
 				});
+				if (content.manifestEntry) {
+					content.manifestEntry.references = resourceRequests.length;
+				}
 				if (!this.cancelled) {
 					const extension = util.getContentTypeExtension(content.contentType) || util.getFilenameExtension(resourceURL, options.filenameReplacedCharacters, options.filenameReplacementCharacter, options.filenameReplacementCharacters);
 					resourceRequests.forEach(callbacks => {
@@ -451,6 +456,10 @@ const UTF8_CHARSET = "utf-8";
 const TAINTED_CANVAS_WARNING_MESSAGE = "SingleFile: canvas elements tainted by a cross-origin resource, dropped from the page:";
 const EMPTY_RESOURCE = "data:,";
 const ORDER_DEPENDENT_RULE = /@(layer|import)\b/i;
+const MANIFEST_TOP_WINDOW_ID = "0";
+const WINDOW_ID_SEPARATOR = ".";
+const MANIFEST_EXCLUDED_OPTION_NAMES = ["rootDocument", "saveFilenameTemplateData", "omitReferrerInTemplateData", "manifest"];
+const MANIFEST_STRING_OPTION_NAMES = ["customStylesheet", "filenameTemplate"];
 
 class Processor {
 	constructor(options, processorHelper, batchRequest) {
@@ -471,7 +480,10 @@ class Processor {
 			frames: new Map()
 		};
 		this.fontTests = options.fontTests;
+		this.frameManifests = [];
+		this.startDate = new Date();
 		options.generatedDataURIs = new Set();
+		options.manifestEntries = options.manifest ? [] : undefined;
 	}
 
 	registerGeneratedDataURI(dataURI) {
@@ -504,7 +516,8 @@ class Processor {
 				resourceReferrer: this.options.resourceReferrer,
 				expectedType: "document",
 				acceptHeaders: this.options.acceptHeaders,
-				networkTimeout: this.options.networkTimeout
+				networkTimeout: this.options.networkTimeout,
+				manifestEntries: this.options.manifestEntries
 			});
 			pageContent = content.data || "";
 		}
@@ -654,12 +667,52 @@ class Processor {
 		if (this.options.addProof) {
 			pageData.hash = await util.digest("SHA-256", content);
 		}
+		if (this.options.manifest) {
+			pageData.manifest = this.getManifest();
+		}
 		if (this.options.retrieveLinks) {
 			pageData.links = Array.from(new Set(Array.from(this.doc.links).map(linkElement => linkElement.href)));
 		}
 		return pageData;
 	}
 
+	getManifest() {
+		const windowId = this.options.windowId || MANIFEST_TOP_WINDOW_ID;
+		const manifestData = this.options.manifestData || {};
+		const frame = {
+			id: windowId,
+			parent: windowId.includes(WINDOW_ID_SEPARATOR) ? windowId.substring(0, windowId.lastIndexOf(WINDOW_ID_SEPARATOR)) : null,
+			url: this.baseURI
+		};
+		if (manifestData.lastModified) {
+			frame.lastModified = manifestData.lastModified;
+		}
+		if (manifestData.navigation) {
+			frame.navigation = manifestData.navigation;
+		}
+		frame.resources = this.options.manifestEntries.map(entry => {
+			const timing = manifestData.timings && (manifestData.timings[entry.finalUrl] || manifestData.timings[entry.url]);
+			return timing ? Object.assign({}, entry, { timing }) : entry;
+		});
+		const frames = [frame, ...this.frameManifests];
+		if (!this.options.rootDocument) {
+			return { frames };
+		}
+		const manifest = {
+			subject: { url: this.options.saveUrl, finalUrl: manifestData.url || this.baseURI, title: this.doc.title },
+			time: { captureStarted: this.startDate.toISOString(), captureEnded: new Date().toISOString() }
+		};
+		if (manifestData.timeOrigin) {
+			manifest.time.loaded = manifestData.timeOrigin;
+		}
+		if (manifestData.environment) {
+			manifest.environment = manifestData.environment;
+		}
+		manifest.options = getManifestOptions(this.options);
+		manifest.frames = frames;
+		return manifest;
+	}
+
 	preProcessPage() {
 		this.doc.querySelectorAll("style[" + util.CUSTOM_STYLESHEET_ATTRIBUTE_NAME + "]").forEach(element => element.removeAttribute(util.CUSTOM_STYLESHEET_ATTRIBUTE_NAME));
 		this.doc.body.querySelectorAll(":not(svg) title, meta, link[href][rel*=\"icon\"]").forEach(element => {
@@ -1463,6 +1516,7 @@ class Processor {
 			options.scrollPosition = frameData.scrollPosition;
 			options.scrolling = frameData.scrolling;
 			options.adoptedStyleSheets = frameData.adoptedStyleSheets;
+			options.manifestData = frameData.manifestData;
 			frameData.runner = new Runner(options, processorHelper);
 			frameData.frameElement = frameElement;
 			await frameData.runner.loadPage();
@@ -1668,6 +1722,9 @@ class Processor {
 					const { frameElement, pageData, frameWindowId, frameData } = capturedFrame;
 					this.processorHelper.processFrame(frameElement, pageData, this.options, this.resources, frameWindowId, frameData);
 					this.stats.addAll(pageData);
+					if (pageData.manifest) {
+						this.frameManifests.push(...pageData.manifest.frames);
+					}
 				}
 			});
 		}
@@ -1702,7 +1759,8 @@ class Processor {
 				this.doc.head.appendChild(styleElement);
 			}
 			let stylesheetContent = "";
-			cssVariables.forEach(({ content, url }, indexResource) => {
+			const variables = Array.from(cssVariables.entries()).sort(([firstIndex], [secondIndex]) => firstIndex - secondIndex);
+			variables.forEach(([indexResource, { content, url }]) => {
 				cssVariables.delete(indexResource);
 				if (stylesheetContent) {
 					stylesheetContent += ";";
@@ -1911,6 +1969,18 @@ function nameAssignedElements(hostElement) {
 	return splitSlots;
 }
 
+function getManifestOptions(options) {
+	const manifestOptions = {};
+	Object.keys(options).sort().forEach(name => {
+		const value = options[name];
+		const primitive = typeof value == "boolean" || typeof value == "number";
+		if ((primitive && !MANIFEST_EXCLUDED_OPTION_NAMES.includes(name)) || (typeof value == "string" && MANIFEST_STRING_OPTION_NAMES.includes(name))) {
+			manifestOptions[name] = value;
+		}
+	});
+	return manifestOptions;
+}
+
 function removeInsertedParagraphs(doc) {
 	getInsertedParagraphs(doc).forEach((_, paragraph) => paragraph.remove());
 }

package/core/lib/processor-helper-common.js

@@ -312,7 +312,8 @@ class ProcessorHelperCommon {
 			blockMixedContent: options.blockMixedContent,
 			expectedType: "stylesheet",
 			acceptHeaders: options.acceptHeaders,
-			networkTimeout: options.networkTimeout
+			networkTimeout: options.networkTimeout,
+			manifestEntries: options.manifestEntries
 		});
 		if (!(matchCharsetEquals(content.data, content.charset) || matchCharsetEquals(content.data, options.charset))) {
 			options = Object.assign({}, options, { charset: getCharset(content.data) });

package/core/lib/processor-helper-inline.js

@@ -263,7 +263,8 @@ function getProcessorHelperClass(utilInstance) {
 					blockMixedContent: options.blockMixedContent,
 					expectedType: "stylesheet",
 					acceptHeaders: options.acceptHeaders,
-					networkTimeout: options.networkTimeout
+					networkTimeout: options.networkTimeout,
+					manifestEntries: options.manifestEntries
 				});
 				if (content.failed) {
 					const liveContent = this.getLiveStylesheetContent(element, options);
@@ -405,7 +406,8 @@ function getProcessorHelperClass(utilInstance) {
 													resourceReferrer: options.resourceReferrer,
 													referrerPolicy: getReferrerPolicy(resourceElement),
 													acceptHeaders: options.acceptHeaders,
-													networkTimeout: options.networkTimeout
+													networkTimeout: options.networkTimeout,
+													manifestEntries: options.manifestEntries
 												})).data;
 												// eslint-disable-next-line no-unused-vars
 											} catch (error) {

package/core/lib/processor-helper.js

@@ -288,7 +288,8 @@ function getProcessorHelperClass(utilInstance) {
 					blockMixedContent: options.blockMixedContent,
 					expectedType: "stylesheet",
 					acceptHeaders: options.acceptHeaders,
-					networkTimeout: options.networkTimeout
+					networkTimeout: options.networkTimeout,
+					manifestEntries: options.manifestEntries
 				});
 				if (content.failed) {
 					const liveContent = this.getLiveStylesheetContent(element, options);
@@ -477,7 +478,8 @@ function getProcessorHelperClass(utilInstance) {
 													resourceReferrer: options.resourceReferrer,
 													referrerPolicy: getReferrerPolicy(resourceElement),
 													acceptHeaders: options.acceptHeaders,
-													networkTimeout: options.networkTimeout
+													networkTimeout: options.networkTimeout,
+													manifestEntries: options.manifestEntries
 												})).data;
 												// eslint-disable-next-line no-unused-vars
 											} catch (error) {

package/core/util.js

@@ -72,6 +72,15 @@ const CONTENT_TYPES_HTML = ["text/html", "application/xhtml+xml"];
 const EXPECTED_TYPES_MEDIA = ["font", "image", "video", "audio", "model"];
 const EXPECTED_TYPE_IMAGE = "image";
 const EXPECTED_TYPE_DOCUMENT = "document";
+const DATA_URI_PREFIX = "data:";
+const MANIFEST_OUTCOME_EMBEDDED = "embedded";
+const MANIFEST_OUTCOME_FAILED = "failed";
+const MANIFEST_OUTCOME_BLOCKED = "blocked";
+const MANIFEST_OUTCOME_REJECTED = "rejected";
+const MANIFEST_OUTCOME_OVERSIZED = "oversized";
+const MANIFEST_FETCHED_BY_PAGE = "page";
+const MANIFEST_FETCHED_BY_FRAME = "frame";
+const MANIFEST_HEADER_NAMES = ["date", "last-modified", "etag", "content-length", "content-encoding", "cache-control", "age", "expires", "server"];
 
 const URL = globalThis.URL;
 const DOMParser = globalThis.DOMParser;
@@ -246,6 +255,16 @@ function getInstance(utilOptions) {
 	};
 
 	async function getContent(resourceURL, options) {
+		const entry = options.manifestEntries && !resourceURL.startsWith(DATA_URI_PREFIX) ? { url: resourceURL, role: options.expectedType || EXPECTED_TYPE_DOCUMENT, outcome: MANIFEST_OUTCOME_FAILED } : undefined;
+		const content = await fetchContent(resourceURL, options, entry);
+		if (entry) {
+			options.manifestEntries.push(entry);
+			content.manifestEntry = entry;
+		}
+		return content;
+	}
+
+	async function fetchContent(resourceURL, options, entry) {
 		let response, startTime, networkTimeoutId, networkTimeoutPromise, resolveNetworkTimeoutPromise;
 		const fetchResource = utilOptions.fetch;
 		const fetchFrameResource = utilOptions.frameFetch;
@@ -254,6 +273,7 @@ function getInstance(utilOptions) {
 			log("  // STARTED download url =", resourceURL, "asBinary =", options.asBinary);
 		}
 		if (options.blockMixedContent && /^https:/i.test(options.baseURI) && !/^https:/i.test(resourceURL) && !/^blob:https:/i.test(resourceURL)) {
+			setManifestOutcome(entry, MANIFEST_OUTCOME_BLOCKED);
 			return getFetchResponse(resourceURL, options);
 		}
 		if (options.networkTimeout) {
@@ -274,21 +294,24 @@ function getInstance(utilOptions) {
 						fetchFrameResource(resourceURL, { frameId: options.frameId, referrer: options.resourceReferrer, referrerPolicy: options.referrerPolicy || undefined, stylesheetURL: options.stylesheetURL || undefined, stylesheetReferrerPolicy: options.stylesheetReferrerPolicy || undefined, headers: { accept } }),
 						networkTimeoutPromise
 					]);
+					setManifestResponse(entry, response, MANIFEST_FETCHED_BY_FRAME);
 					// eslint-disable-next-line no-unused-vars
 				} catch (error) {
 					response = await Promise.race([
 						fetchResource(resourceURL, { referrer: options.resourceReferrer, referrerPolicy: options.referrerPolicy || undefined, stylesheetURL: options.stylesheetURL || undefined, stylesheetReferrerPolicy: options.stylesheetReferrerPolicy || undefined, headers: { accept } }),
 						networkTimeoutPromise
 					]);
+					setManifestResponse(entry, response, MANIFEST_FETCHED_BY_PAGE);
 				}
 			} else {
 				response = await Promise.race([
 					fetchResource(resourceURL, { referrer: options.resourceReferrer, referrerPolicy: options.referrerPolicy || undefined, stylesheetURL: options.stylesheetURL || undefined, stylesheetReferrerPolicy: options.stylesheetReferrerPolicy || undefined, headers: { accept } }),
 					networkTimeoutPromise
 				]);
+				setManifestResponse(entry, response, MANIFEST_FETCHED_BY_PAGE);
 			}
-			// eslint-disable-next-line no-unused-vars
 		} catch (error) {
+			setManifestError(entry, error);
 			return getFailedFetchResponse(resourceURL, options);
 		} finally {
 			resolveNetworkTimeoutPromise();
@@ -299,10 +322,11 @@ function getInstance(utilOptions) {
 		let buffer;
 		try {
 			buffer = await response.arrayBuffer();
-			// eslint-disable-next-line no-unused-vars
 		} catch (error) {
+			setManifestError(entry, error);
 			return options.inline ? { data: options.asBinary ? helper.EMPTY_RESOURCE : "", resourceURL } : { resourceURL };
 		}
+		await setManifestContent(entry, buffer);
 		resourceURL = response.url || resourceURL;
 		let contentType = "", charset;
 		try {
@@ -322,8 +346,15 @@ function getInstance(utilOptions) {
 		if (!charset && options.charset) {
 			charset = options.charset;
 		}
+		if (entry) {
+			entry.contentType = contentType;
+			if (charset) {
+				entry.charset = charset;
+			}
+		}
 		if (options.asBinary) {
 			if ((response.status >= 400 && options.expectedType != EXPECTED_TYPE_IMAGE) || (EXPECTED_TYPES_MEDIA.includes(options.expectedType) && CONTENT_TYPES_HTML.includes(contentType))) {
+				setManifestOutcome(entry, MANIFEST_OUTCOME_REJECTED);
 				return getFetchResponse(resourceURL, options);
 			}
 			try {
@@ -331,19 +362,23 @@ function getInstance(utilOptions) {
 					log("  // ENDED   download url =", resourceURL, "delay =", Date.now() - startTime);
 				}
 				if (options.maxResourceSizeEnabled && buffer.byteLength > options.maxResourceSize * ONE_MB) {
+					setManifestOutcome(entry, MANIFEST_OUTCOME_OVERSIZED);
 					return getFetchResponse(resourceURL, options);
 				} else {
+					setManifestOutcome(entry, MANIFEST_OUTCOME_EMBEDDED);
 					return getFetchResponse(resourceURL, options, buffer, null, contentType);
 				}
-				// eslint-disable-next-line no-unused-vars
 			} catch (error) {
+				setManifestError(entry, error);
 				return getFetchResponse(resourceURL, options);
 			}
 		} else {
 			if (response.status >= 400 && options.expectedType != EXPECTED_TYPE_DOCUMENT) {
+				setManifestOutcome(entry, MANIFEST_OUTCOME_REJECTED);
 				return getFailedFetchResponse(resourceURL, options);
 			}
 			if (options.validateTextContentType && contentType && !contentType.startsWith(PREFIX_CONTENT_TYPE_TEXT)) {
+				setManifestOutcome(entry, MANIFEST_OUTCOME_REJECTED);
 				return getFetchResponse(resourceURL, options);
 			}
 			if (!charset) {
@@ -353,14 +388,16 @@ function getInstance(utilOptions) {
 				log("  // ENDED   download url =", resourceURL, "delay =", Date.now() - startTime);
 			}
 			if (options.maxResourceSizeEnabled && buffer.byteLength > options.maxResourceSize * ONE_MB) {
+				setManifestOutcome(entry, MANIFEST_OUTCOME_OVERSIZED);
 				return getFetchResponse(resourceURL, options, null, charset);
 			} else {
 				try {
 					const fetchResponse = await getFetchResponse(resourceURL, options, buffer, charset, contentType);
 					fetchResponse.referrerPolicy = (response.headers && response.headers.get("referrer-policy")) || undefined;
+					setManifestOutcome(entry, MANIFEST_OUTCOME_EMBEDDED);
 					return fetchResponse;
-					// eslint-disable-next-line no-unused-vars
 				} catch (error) {
+					setManifestError(entry, error);
 					return getFetchResponse(resourceURL, options, null, charset);
 				}
 			}
@@ -368,6 +405,60 @@ function getInstance(utilOptions) {
 	}
 }
 
+function setManifestResponse(entry, response, fetchedBy) {
+	if (entry) {
+		entry.fetchedBy = fetchedBy;
+		if (response.url) {
+			entry.finalUrl = response.url;
+		}
+		if (response.status !== undefined) {
+			entry.status = response.status;
+		}
+		const headers = {};
+		MANIFEST_HEADER_NAMES.forEach(name => {
+			const value = getHeaderValue(response.headers, name);
+			if (value) {
+				headers[name] = value;
+			}
+		});
+		if (Object.keys(headers).length) {
+			entry.headers = headers;
+		}
+	}
+}
+
+function getHeaderValue(headers, name) {
+	if (headers) {
+		if (typeof headers.get == "function") {
+			return headers.get(name);
+		}
+		const key = Object.keys(headers).find(key => key.toLowerCase() == name);
+		return key === undefined ? undefined : headers[key];
+	}
+}
+
+async function setManifestContent(entry, buffer) {
+	if (entry) {
+		entry.size = buffer.byteLength;
+		entry.sha256 = await helper.digest("SHA-256", buffer);
+	}
+}
+
+function setManifestOutcome(entry, outcome) {
+	if (entry) {
+		entry.outcome = outcome;
+	}
+}
+
+function setManifestError(entry, error) {
+	if (entry) {
+		entry.outcome = MANIFEST_OUTCOME_FAILED;
+		if (error && error.message) {
+			entry.error = error.message;
+		}
+	}
+}
+
 async function getFailedFetchResponse(resourceURL, options) {
 	const response = await getFetchResponse(resourceURL, options);
 	response.failed = true;
@@ -398,7 +489,7 @@ async function getFetchResponse(resourceURL, options, data, charset, contentType
 				charset = "utf-8";
 				data = new TextDecoder(charset).decode(data);
 			}
-			data = data.replace(/\ufeff/gi, "");
+			data = data.replace(/^\ufeff/, "");
 		}
 	} else if (options.inline) {
 		data = options.asBinary ? helper.EMPTY_RESOURCE : "";

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-core",
-	"version": "1.6.25",
+	"version": "1.6.27",
 	"description": "SingleFile Core",
 	"author": "Gildas Lormeau",
 	"license": "AGPL-3.0-or-later",

package/processors/frame-tree/content/content-frame-tree.js

@@ -237,6 +237,7 @@ function initResponse(message) {
 				frameData.scrollPosition = messageFrameData.scrollPosition;
 				frameData.scrolling = messageFrameData.scrolling;
 				frameData.adoptedStyleSheets = messageFrameData.adoptedStyleSheets;
+				frameData.manifestData = messageFrameData.manifestData;
 			}
 		});
 		const remainingFrames = windowData.frames.filter(frameData => !frameData.processed).length;
@@ -459,6 +460,7 @@ function getFrameData(document, win, windowId, options, scrolling) {
 		scrollPosition: docData.scrollPosition,
 		scrolling,
 		adoptedStyleSheets: docData.adoptedStyleSheets,
+		manifestData: docData.manifestData,
 		processed: true
 	};
 }

package/single-file.js

@@ -131,5 +131,14 @@ async function getPageData(options = {}, initOptions, doc, win) {
 		}
 		pageData.content = Array.from(new Uint8Array(arrayBuffer));
 	}
+	if (pageData.manifest) {
+		const content = options.compressContent ? new Uint8Array(pageData.content) : pageData.content;
+		pageData.manifest.output = {
+			filename: pageData.filename,
+			mimeType: pageData.mimeType,
+			size: options.compressContent ? content.length : helper.getContentSize(content),
+			sha256: await helper.digest("SHA-256", content)
+		};
+	}
 	return pageData;
 }