single-file-core 1.1.78 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/constants.js +3 -1
- package/core/helper.js +9 -2
- package/core/index.js +61 -751
- package/core/infobar.js +1 -1
- package/core/lib/processor-helper-common.js +323 -0
- package/core/lib/processor-helper-inline.js +876 -0
- package/core/lib/processor-helper.js +801 -0
- package/core/processor-helper.js +34 -0
- package/core/util.js +48 -13
- package/modules/css-fonts-minifier.js +41 -18
- package/modules/index.js +0 -2
- package/package.json +1 -1
- package/processors/compression/compression-display.js +74 -0
- package/processors/compression/compression-extract.js +151 -0
- package/processors/compression/compression.js +305 -0
- package/processors/hooks/content/content-hooks-frames.js +4 -3
- package/processors/index.js +2 -0
- package/single-file-infobar.js +1 -0
- package/single-file.js +24 -2
- package/vendor/index.js +2 -2
- package/vendor/zip/z-worker.js +1 -0
- package/vendor/zip/zip.js +4955 -0
- package/vendor/zip/zip.min.js +1 -0
- package/modules/css-fonts-alt-minifier.js +0 -343
package/core/index.js
CHANGED
|
@@ -23,36 +23,40 @@
|
|
|
23
23
|
|
|
24
24
|
/* global globalThis */
|
|
25
25
|
|
|
26
|
+
import {
|
|
27
|
+
getProcessorHelperClass,
|
|
28
|
+
cssTree
|
|
29
|
+
} from "./processor-helper.js";
|
|
30
|
+
|
|
26
31
|
const DEBUG = false;
|
|
27
32
|
|
|
28
33
|
const Set = globalThis.Set;
|
|
29
34
|
const Map = globalThis.Map;
|
|
30
35
|
const JSON = globalThis.JSON;
|
|
31
|
-
const setTimeout = globalThis.setTimeout;
|
|
32
|
-
const clearTimeout = globalThis.clearTimeout;
|
|
33
|
-
const Image = globalThis.Image;
|
|
34
36
|
|
|
35
|
-
let util
|
|
37
|
+
let util;
|
|
36
38
|
|
|
37
39
|
export {
|
|
38
40
|
getClass
|
|
39
41
|
};
|
|
40
42
|
|
|
41
43
|
function getClass(...args) {
|
|
42
|
-
[util
|
|
44
|
+
[util] = args;
|
|
43
45
|
return SingleFileClass;
|
|
44
46
|
}
|
|
45
47
|
|
|
46
48
|
class SingleFileClass {
|
|
47
49
|
constructor(options) {
|
|
48
50
|
this.options = options;
|
|
51
|
+
const ProcessorHelper = getProcessorHelperClass(options, util);
|
|
52
|
+
this.processorHelper = new ProcessorHelper();
|
|
49
53
|
}
|
|
50
54
|
async run() {
|
|
51
55
|
const waitForUserScript = globalThis[util.WAIT_FOR_USERSCRIPT_PROPERTY_NAME];
|
|
52
56
|
if (this.options.userScriptEnabled && waitForUserScript) {
|
|
53
57
|
await waitForUserScript(util.ON_BEFORE_CAPTURE_EVENT_NAME);
|
|
54
58
|
}
|
|
55
|
-
this.runner = new Runner(this.options, true);
|
|
59
|
+
this.runner = new Runner(this.options, this.processorHelper, true);
|
|
56
60
|
await this.runner.loadPage();
|
|
57
61
|
await this.runner.initialize();
|
|
58
62
|
if (this.options.userScriptEnabled && waitForUserScript) {
|
|
@@ -124,7 +128,7 @@ const STAGES = [{
|
|
|
124
128
|
],
|
|
125
129
|
parallel: [
|
|
126
130
|
{ option: "blockVideos", action: "insertMissingVideoPosters" },
|
|
127
|
-
{ action: "
|
|
131
|
+
{ action: "resolveStylesheetsURLs" },
|
|
128
132
|
{ option: "!removeFrames", action: "resolveFrameURLs" }
|
|
129
133
|
]
|
|
130
134
|
}, {
|
|
@@ -162,7 +166,7 @@ const STAGES = [{
|
|
|
162
166
|
}];
|
|
163
167
|
|
|
164
168
|
class Runner {
|
|
165
|
-
constructor(options, root) {
|
|
169
|
+
constructor(options, processorHelper, root) {
|
|
166
170
|
const rootDocDefined = root && options.doc;
|
|
167
171
|
this.root = root;
|
|
168
172
|
this.options = options;
|
|
@@ -174,7 +178,7 @@ class Runner {
|
|
|
174
178
|
this.options.updatedResources = this.options.updatedResources || {};
|
|
175
179
|
this.options.fontTests = new Map();
|
|
176
180
|
this.batchRequest = new BatchRequest();
|
|
177
|
-
this.processor = new Processor(options, this.batchRequest);
|
|
181
|
+
this.processor = new Processor(options, processorHelper, this.batchRequest);
|
|
178
182
|
if (rootDocDefined) {
|
|
179
183
|
const docData = util.preProcessDoc(this.options.doc, this.options.win, this.options);
|
|
180
184
|
this.options.canvases = docData.canvases;
|
|
@@ -360,6 +364,7 @@ class BatchRequest {
|
|
|
360
364
|
indexResource = indexResource + 1;
|
|
361
365
|
const content = await util.getContent(resourceURL, {
|
|
362
366
|
asBinary,
|
|
367
|
+
inline: !options.compressContent,
|
|
363
368
|
expectedType,
|
|
364
369
|
maxResourceSize: options.maxResourceSize,
|
|
365
370
|
maxResourceSizeEnabled: options.maxResourceSizeEnabled,
|
|
@@ -372,10 +377,11 @@ class BatchRequest {
|
|
|
372
377
|
});
|
|
373
378
|
onloadListener({ url: resourceURL });
|
|
374
379
|
if (!this.cancelled) {
|
|
380
|
+
const extension = util.getContentTypeExtension(content.contentType) || util.getFilenameExtension(resourceURL, options.filenameReplacedCharacters, options.filenameReplacementCharacter);
|
|
375
381
|
resourceRequests.forEach(callbacks => {
|
|
376
382
|
const duplicateCallbacks = this.duplicates.get(requestKey);
|
|
377
383
|
const duplicate = duplicateCallbacks && duplicateCallbacks.length > 1 && duplicateCallbacks.includes(callbacks);
|
|
378
|
-
callbacks.resolve({ content: content.data, indexResource: currentIndexResource, duplicate });
|
|
384
|
+
callbacks.resolve({ content: content.data, indexResource: currentIndexResource, duplicate, contentType: content.contentType, extension });
|
|
379
385
|
});
|
|
380
386
|
}
|
|
381
387
|
} catch (error) {
|
|
@@ -401,24 +407,27 @@ class BatchRequest {
|
|
|
401
407
|
// ---------
|
|
402
408
|
// Processor
|
|
403
409
|
// ---------
|
|
404
|
-
const
|
|
405
|
-
const PREFIX_DATA_URI_IMAGE_SVG = "data:image/svg+xml";
|
|
406
|
-
const SCRIPT_TAG_FOUND = /<script/gi;
|
|
407
|
-
const NOSCRIPT_TAG_FOUND = /<noscript/gi;
|
|
408
|
-
const CANVAS_TAG_FOUND = /<canvas/gi;
|
|
409
|
-
const SHADOWROOT_ATTRIBUTE_NAME = "shadowroot";
|
|
410
|
+
const SHADOWROOT_ATTRIBUTE_NAME = "shadowrootmode";
|
|
410
411
|
const SCRIPT_TEMPLATE_SHADOW_ROOT = "data-template-shadow-root";
|
|
411
412
|
const UTF8_CHARSET = "utf-8";
|
|
412
413
|
|
|
413
414
|
class Processor {
|
|
414
|
-
constructor(options, batchRequest) {
|
|
415
|
+
constructor(options, processorHelper, batchRequest) {
|
|
415
416
|
this.options = options;
|
|
417
|
+
this.processorHelper = processorHelper;
|
|
416
418
|
this.stats = new Stats(options);
|
|
417
419
|
this.baseURI = normalizeURL(options.baseURI || options.url);
|
|
418
420
|
this.batchRequest = batchRequest;
|
|
419
421
|
this.stylesheets = new Map();
|
|
420
422
|
this.styles = new Map();
|
|
421
|
-
this.
|
|
423
|
+
this.resources = {
|
|
424
|
+
cssVariables: new Map(),
|
|
425
|
+
fonts: new Map(),
|
|
426
|
+
stylesheets: new Map(),
|
|
427
|
+
scripts: new Map(),
|
|
428
|
+
images: new Map(),
|
|
429
|
+
frames: new Map()
|
|
430
|
+
};
|
|
422
431
|
this.fontTests = options.fontTests;
|
|
423
432
|
}
|
|
424
433
|
|
|
@@ -439,6 +448,7 @@ class Processor {
|
|
|
439
448
|
let content;
|
|
440
449
|
if (!pageContent || this.options.saveRawPage) {
|
|
441
450
|
content = await util.getContent(this.baseURI, {
|
|
451
|
+
inline: !this.options.compressContent,
|
|
442
452
|
maxResourceSize: this.options.maxResourceSize,
|
|
443
453
|
maxResourceSizeEnabled: this.options.maxResourceSizeEnabled,
|
|
444
454
|
charset,
|
|
@@ -532,10 +542,10 @@ class Processor {
|
|
|
532
542
|
}
|
|
533
543
|
}
|
|
534
544
|
if (this.options.insertMetaCSP) {
|
|
535
|
-
const
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
this.doc.head.appendChild(
|
|
545
|
+
const metaElement = this.doc.createElement("meta");
|
|
546
|
+
metaElement.httpEquiv = "content-security-policy";
|
|
547
|
+
this.processorHelper.setMetaCSP(metaElement);
|
|
548
|
+
this.doc.head.appendChild(metaElement);
|
|
539
549
|
}
|
|
540
550
|
if (this.options.insertMetaNoIndex) {
|
|
541
551
|
let metaElement = this.doc.querySelector("meta[name=robots][content*=noindex]");
|
|
@@ -561,12 +571,13 @@ class Processor {
|
|
|
561
571
|
}
|
|
562
572
|
const filename = await util.formatFilename(content, this.options);
|
|
563
573
|
const matchTitle = this.baseURI.match(/([^/]*)\/?(\.html?.*)$/) || this.baseURI.match(/\/\/([^/]*)\/?$/);
|
|
564
|
-
const
|
|
574
|
+
const additionalData = this.processorHelper.getAdditionalPageData(this.doc, content, this.resources);
|
|
575
|
+
const pageData = Object.assign({
|
|
565
576
|
stats: this.stats.data,
|
|
566
577
|
title: this.options.title || (this.baseURI && matchTitle ? matchTitle[1] : url.hostname ? url.hostname : ""),
|
|
567
578
|
filename,
|
|
568
579
|
content
|
|
569
|
-
};
|
|
580
|
+
}, additionalData);
|
|
570
581
|
if (this.options.addProof) {
|
|
571
582
|
pageData.hash = await util.digest("SHA-256", content);
|
|
572
583
|
}
|
|
@@ -778,13 +789,13 @@ class Processor {
|
|
|
778
789
|
const noscriptPlaceholders = new Map();
|
|
779
790
|
this.doc.querySelectorAll("noscript").forEach(noscriptElement => {
|
|
780
791
|
const placeholderElement = this.doc.createElement("div");
|
|
781
|
-
placeholderElement.innerHTML = noscriptElement.dataset.
|
|
792
|
+
placeholderElement.innerHTML = noscriptElement.dataset[util.NO_SCRIPT_PROPERTY_NAME];
|
|
782
793
|
noscriptElement.replaceWith(placeholderElement);
|
|
783
794
|
noscriptPlaceholders.set(placeholderElement, noscriptElement);
|
|
784
795
|
});
|
|
785
796
|
this.doc.querySelectorAll("meta[http-equiv=refresh], meta[disabled-http-equiv]").forEach(element => element.remove());
|
|
786
797
|
noscriptPlaceholders.forEach((noscriptElement, placeholderElement) => {
|
|
787
|
-
noscriptElement.dataset.
|
|
798
|
+
noscriptElement.dataset[util.NO_SCRIPT_PROPERTY_NAME] = placeholderElement.innerHTML;
|
|
788
799
|
placeholderElement.replaceWith(noscriptElement);
|
|
789
800
|
});
|
|
790
801
|
this.doc.querySelectorAll("meta[http-equiv=\"content-security-policy\"]").forEach(element => element.remove());
|
|
@@ -804,7 +815,8 @@ class Processor {
|
|
|
804
815
|
element.remove();
|
|
805
816
|
}
|
|
806
817
|
});
|
|
807
|
-
this.
|
|
818
|
+
this.processorHelper.removeUnusedStylesheets(this.doc);
|
|
819
|
+
this.doc.querySelectorAll("link[rel*=stylesheet]:not([href]),link[rel*=stylesheet][href=\"\"]").forEach(element => element.remove());
|
|
808
820
|
if (this.options.removeHiddenElements) {
|
|
809
821
|
this.doc.querySelectorAll("input[type=hidden]").forEach(element => element.remove());
|
|
810
822
|
}
|
|
@@ -918,7 +930,7 @@ class Processor {
|
|
|
918
930
|
if (canvasData.backgroundColor) {
|
|
919
931
|
backgroundStyle["background-color"] = canvasData.backgroundColor;
|
|
920
932
|
}
|
|
921
|
-
|
|
933
|
+
this.processorHelper.setBackgroundImage(canvasElement, "url(" + canvasData.dataURI + ")", backgroundStyle);
|
|
922
934
|
this.stats.add("processed", "canvas", 1);
|
|
923
935
|
}
|
|
924
936
|
}
|
|
@@ -1062,13 +1074,13 @@ class Processor {
|
|
|
1062
1074
|
} else {
|
|
1063
1075
|
const styleContent = element.getAttribute("style");
|
|
1064
1076
|
const declarationList = cssTree.parse(styleContent, { context: "declarationList", parseCustomProperty: true });
|
|
1065
|
-
|
|
1077
|
+
this.processorHelper.resolveStylesheetURLs(declarationList, this.baseURI, this.workStyleElement);
|
|
1066
1078
|
this.styles.set(element, declarationList);
|
|
1067
1079
|
}
|
|
1068
1080
|
});
|
|
1069
1081
|
}
|
|
1070
1082
|
|
|
1071
|
-
async
|
|
1083
|
+
async resolveStylesheetsURLs() {
|
|
1072
1084
|
await Promise.all(Array.from(this.doc.querySelectorAll("style, link[rel*=stylesheet]")).map(async element => {
|
|
1073
1085
|
const options = Object.assign({}, this.options, { charset: this.charset });
|
|
1074
1086
|
let mediaText;
|
|
@@ -1080,10 +1092,7 @@ class Processor {
|
|
|
1080
1092
|
mediaText,
|
|
1081
1093
|
scoped
|
|
1082
1094
|
};
|
|
1083
|
-
|
|
1084
|
-
options.charset = element.charset;
|
|
1085
|
-
}
|
|
1086
|
-
await processElement(element, stylesheetInfo, this.stylesheets, this.baseURI, options, this.workStyleElement);
|
|
1095
|
+
await this.processorHelper.processLinkElement(element, stylesheetInfo, this.stylesheets, this.baseURI, options, this.workStyleElement, this.resources);
|
|
1087
1096
|
}));
|
|
1088
1097
|
if (this.options.rootDocument) {
|
|
1089
1098
|
const newResources = Object.keys(this.options.updatedResources)
|
|
@@ -1096,38 +1105,14 @@ class Processor {
|
|
|
1096
1105
|
const element = this.doc.createElement("style");
|
|
1097
1106
|
this.doc.body.appendChild(element);
|
|
1098
1107
|
element.textContent = resource.content;
|
|
1099
|
-
await
|
|
1108
|
+
await this.processorHelper.processStylesheetElement(element, stylesheetInfo, this.stylesheets, this.baseURI, this.options, this.workStyleElement, this.resources);
|
|
1100
1109
|
}
|
|
1101
1110
|
}));
|
|
1102
1111
|
}
|
|
1103
|
-
|
|
1104
|
-
async function processElement(element, stylesheetInfo, stylesheets, baseURI, options, workStyleElement) {
|
|
1105
|
-
let stylesheet;
|
|
1106
|
-
stylesheets.set(element, stylesheetInfo);
|
|
1107
|
-
if (!options.blockStylesheets) {
|
|
1108
|
-
if (element.tagName.toUpperCase() == "LINK") {
|
|
1109
|
-
stylesheet = await ProcessorHelper.resolveLinkStylesheetURLs(element.href, baseURI, options, workStyleElement);
|
|
1110
|
-
} else {
|
|
1111
|
-
stylesheet = cssTree.parse(element.textContent, { context: "stylesheet", parseCustomProperty: true });
|
|
1112
|
-
const importFound = await ProcessorHelper.resolveImportURLs(stylesheet, baseURI, options, workStyleElement);
|
|
1113
|
-
if (importFound) {
|
|
1114
|
-
stylesheet = cssTree.parse(cssTree.generate(stylesheet), { context: "stylesheet", parseCustomProperty: true });
|
|
1115
|
-
}
|
|
1116
|
-
}
|
|
1117
|
-
}
|
|
1118
|
-
if (stylesheet && stylesheet.children) {
|
|
1119
|
-
if (options.compressCSS) {
|
|
1120
|
-
ProcessorHelper.removeSingleLineCssComments(stylesheet);
|
|
1121
|
-
}
|
|
1122
|
-
ProcessorHelper.replacePseudoClassDefined(stylesheet);
|
|
1123
|
-
stylesheetInfo.stylesheet = stylesheet;
|
|
1124
|
-
} else {
|
|
1125
|
-
stylesheets.delete(element);
|
|
1126
|
-
}
|
|
1127
|
-
}
|
|
1128
1112
|
}
|
|
1129
1113
|
|
|
1130
1114
|
async resolveFrameURLs() {
|
|
1115
|
+
const processorHelper = this.processorHelper;
|
|
1131
1116
|
if (!this.options.saveRawPage) {
|
|
1132
1117
|
const frameElements = Array.from(this.doc.querySelectorAll("iframe, frame, object[type=\"text/html\"][data]"));
|
|
1133
1118
|
await Promise.all(frameElements.map(async frameElement => {
|
|
@@ -1173,7 +1158,7 @@ class Processor {
|
|
|
1173
1158
|
options.scrollPosition = frameData.scrollPosition;
|
|
1174
1159
|
options.scrolling = frameData.scrolling;
|
|
1175
1160
|
options.adoptedStyleSheets = frameData.adoptedStyleSheets;
|
|
1176
|
-
frameData.runner = new Runner(options);
|
|
1161
|
+
frameData.runner = new Runner(options, processorHelper);
|
|
1177
1162
|
frameData.frameElement = frameElement;
|
|
1178
1163
|
await frameData.runner.loadPage();
|
|
1179
1164
|
await frameData.runner.initialize();
|
|
@@ -1248,47 +1233,19 @@ class Processor {
|
|
|
1248
1233
|
}
|
|
1249
1234
|
|
|
1250
1235
|
async processStylesheets() {
|
|
1251
|
-
this.options.fontDeclarations = new Map();
|
|
1252
1236
|
await Promise.all([...this.stylesheets].map(([, stylesheetInfo]) =>
|
|
1253
|
-
|
|
1237
|
+
this.processorHelper.processStylesheet(stylesheetInfo.stylesheet.children, this.baseURI, this.options, this.resources, this.batchRequest)
|
|
1254
1238
|
));
|
|
1255
1239
|
}
|
|
1256
1240
|
|
|
1257
1241
|
async processStyleAttributes() {
|
|
1258
1242
|
return Promise.all([...this.styles].map(([, stylesheet]) =>
|
|
1259
|
-
|
|
1243
|
+
this.processorHelper.processStyle(stylesheet, this.options, this.resources, this.batchRequest)
|
|
1260
1244
|
));
|
|
1261
1245
|
}
|
|
1262
1246
|
|
|
1263
1247
|
async processPageResources() {
|
|
1264
|
-
|
|
1265
|
-
["link[href][rel*=\"icon\"]", "href", false, true],
|
|
1266
|
-
["object[type=\"image/svg+xml\"], object[type=\"image/svg-xml\"], object[data*=\".svg\"]", "data"],
|
|
1267
|
-
["img[src], input[src][type=image]", "src", true],
|
|
1268
|
-
["embed[src*=\".svg\"]", "src"],
|
|
1269
|
-
["video[poster]", "poster"],
|
|
1270
|
-
["*[background]", "background"],
|
|
1271
|
-
["image", "xlink:href"],
|
|
1272
|
-
["image", "href"]
|
|
1273
|
-
];
|
|
1274
|
-
if (this.options.blockImages) {
|
|
1275
|
-
this.doc.querySelectorAll("svg").forEach(element => element.remove());
|
|
1276
|
-
}
|
|
1277
|
-
let resourcePromises = processAttributeArgs.map(([selector, attributeName, processDuplicates, removeElementIfMissing]) =>
|
|
1278
|
-
ProcessorHelper.processAttribute(this.doc.querySelectorAll(selector), attributeName, this.baseURI, this.options, "image", this.cssVariables, this.styles, this.batchRequest, processDuplicates, removeElementIfMissing)
|
|
1279
|
-
);
|
|
1280
|
-
resourcePromises = resourcePromises.concat([
|
|
1281
|
-
ProcessorHelper.processXLinks(this.doc.querySelectorAll("use"), this.doc, this.baseURI, this.options, this.batchRequest),
|
|
1282
|
-
ProcessorHelper.processSrcset(this.doc.querySelectorAll("img[srcset], source[srcset]"), this.baseURI, this.options, this.batchRequest)
|
|
1283
|
-
]);
|
|
1284
|
-
resourcePromises.push(ProcessorHelper.processAttribute(this.doc.querySelectorAll("object[data*=\".pdf\"]"), "data", this.baseURI, this.options, null, this.cssVariables, this.styles, this.batchRequest));
|
|
1285
|
-
resourcePromises.push(ProcessorHelper.processAttribute(this.doc.querySelectorAll("embed[src*=\".pdf\"]"), "src", this.baseURI, this.options, null, this.cssVariables, this.styles, this.batchRequest));
|
|
1286
|
-
resourcePromises.push(ProcessorHelper.processAttribute(this.doc.querySelectorAll("audio[src], audio > source[src]"), "src", this.baseURI, this.options, "audio", this.cssVariables, this.styles, this.batchRequest));
|
|
1287
|
-
resourcePromises.push(ProcessorHelper.processAttribute(this.doc.querySelectorAll("video[src], video > source[src]"), "src", this.baseURI, this.options, "video", this.cssVariables, this.styles, this.batchRequest));
|
|
1288
|
-
await Promise.all(resourcePromises);
|
|
1289
|
-
if (this.options.saveFavicon) {
|
|
1290
|
-
ProcessorHelper.processShortcutIcons(this.doc);
|
|
1291
|
-
}
|
|
1248
|
+
await this.processorHelper.processPageResources(this.doc, this.baseURI, this.options, this.resources, this.styles, this.batchRequest);
|
|
1292
1249
|
}
|
|
1293
1250
|
|
|
1294
1251
|
async processScripts() {
|
|
@@ -1309,21 +1266,7 @@ class Processor {
|
|
|
1309
1266
|
}
|
|
1310
1267
|
if (testValidURL(resourceURL)) {
|
|
1311
1268
|
element.removeAttribute("src");
|
|
1312
|
-
|
|
1313
|
-
asBinary: true,
|
|
1314
|
-
charset: this.charset != UTF8_CHARSET && this.charset,
|
|
1315
|
-
maxResourceSize: this.options.maxResourceSize,
|
|
1316
|
-
maxResourceSizeEnabled: this.options.maxResourceSizeEnabled,
|
|
1317
|
-
frameId: this.options.windowId,
|
|
1318
|
-
resourceReferrer: this.options.resourceReferrer,
|
|
1319
|
-
baseURI: this.options.baseURI,
|
|
1320
|
-
blockMixedContent: this.options.blockMixedContent,
|
|
1321
|
-
expectedType: "script",
|
|
1322
|
-
acceptHeaders: this.options.acceptHeaders,
|
|
1323
|
-
networkTimeout: this.options.networkTimeout
|
|
1324
|
-
});
|
|
1325
|
-
content.data = getUpdatedResourceContent(resourceURL, content, this.options);
|
|
1326
|
-
element.setAttribute("src", content.data);
|
|
1269
|
+
await this.processorHelper.processScript(element, resourceURL);
|
|
1327
1270
|
if (element.getAttribute("async") == "async" || element.getAttribute(util.ASYNC_SCRIPT_ATTRIBUTE_NAME) == "") {
|
|
1328
1271
|
element.setAttribute("async", "");
|
|
1329
1272
|
}
|
|
@@ -1340,7 +1283,7 @@ class Processor {
|
|
|
1340
1283
|
}
|
|
1341
1284
|
|
|
1342
1285
|
async removeAlternativeFonts() {
|
|
1343
|
-
await
|
|
1286
|
+
await this.processorHelper.removeAlternativeFonts(this.doc, this.stylesheets, this.resources.fonts, this.options.fontTests);
|
|
1344
1287
|
}
|
|
1345
1288
|
|
|
1346
1289
|
async processFrames() {
|
|
@@ -1357,21 +1300,7 @@ class Processor {
|
|
|
1357
1300
|
await frameData.runner.run();
|
|
1358
1301
|
const pageData = await frameData.runner.getPageData();
|
|
1359
1302
|
frameElement.removeAttribute(util.WIN_ID_ATTRIBUTE_NAME);
|
|
1360
|
-
|
|
1361
|
-
if (pageData.content.match(NOSCRIPT_TAG_FOUND) || pageData.content.match(CANVAS_TAG_FOUND) || pageData.content.match(SCRIPT_TAG_FOUND)) {
|
|
1362
|
-
sandbox += " allow-scripts allow-same-origin";
|
|
1363
|
-
}
|
|
1364
|
-
frameElement.setAttribute("sandbox", sandbox);
|
|
1365
|
-
if (frameElement.tagName.toUpperCase() == "OBJECT") {
|
|
1366
|
-
frameElement.setAttribute("data", "data:text/html," + pageData.content);
|
|
1367
|
-
} else {
|
|
1368
|
-
if (frameElement.tagName.toUpperCase() == "FRAME") {
|
|
1369
|
-
frameElement.setAttribute("src", "data:text/html," + pageData.content.replace(/%/g, "%25").replace(/#/g, "%23"));
|
|
1370
|
-
} else {
|
|
1371
|
-
frameElement.setAttribute("srcdoc", pageData.content);
|
|
1372
|
-
frameElement.removeAttribute("src");
|
|
1373
|
-
}
|
|
1374
|
-
}
|
|
1303
|
+
this.processorHelper.processFrame(frameElement, pageData, this.resources, frameWindowId, frameData);
|
|
1375
1304
|
this.stats.addAll(pageData);
|
|
1376
1305
|
} else {
|
|
1377
1306
|
frameElement.removeAttribute(util.WIN_ID_ATTRIBUTE_NAME);
|
|
@@ -1384,32 +1313,7 @@ class Processor {
|
|
|
1384
1313
|
}
|
|
1385
1314
|
|
|
1386
1315
|
replaceStylesheets() {
|
|
1387
|
-
this.
|
|
1388
|
-
const stylesheetInfo = this.stylesheets.get(styleElement);
|
|
1389
|
-
if (stylesheetInfo) {
|
|
1390
|
-
this.stylesheets.delete(styleElement);
|
|
1391
|
-
styleElement.textContent = generateStylesheetContent(stylesheetInfo.stylesheet, this.options);
|
|
1392
|
-
if (stylesheetInfo.mediaText) {
|
|
1393
|
-
styleElement.media = stylesheetInfo.mediaText;
|
|
1394
|
-
}
|
|
1395
|
-
} else {
|
|
1396
|
-
styleElement.remove();
|
|
1397
|
-
}
|
|
1398
|
-
});
|
|
1399
|
-
this.doc.querySelectorAll("link[rel*=stylesheet]").forEach(linkElement => {
|
|
1400
|
-
const stylesheetInfo = this.stylesheets.get(linkElement);
|
|
1401
|
-
if (stylesheetInfo) {
|
|
1402
|
-
this.stylesheets.delete(linkElement);
|
|
1403
|
-
const styleElement = this.doc.createElement("style");
|
|
1404
|
-
if (stylesheetInfo.mediaText) {
|
|
1405
|
-
styleElement.media = stylesheetInfo.mediaText;
|
|
1406
|
-
}
|
|
1407
|
-
styleElement.textContent = generateStylesheetContent(stylesheetInfo.stylesheet, this.options);
|
|
1408
|
-
linkElement.parentElement.replaceChild(styleElement, linkElement);
|
|
1409
|
-
} else {
|
|
1410
|
-
linkElement.remove();
|
|
1411
|
-
}
|
|
1412
|
-
});
|
|
1316
|
+
this.processorHelper.replaceStylesheets(this.doc, this.stylesheets, this.resources, this.options);
|
|
1413
1317
|
}
|
|
1414
1318
|
|
|
1415
1319
|
replaceStyleAttributes() {
|
|
@@ -1417,7 +1321,7 @@ class Processor {
|
|
|
1417
1321
|
const declarationList = this.styles.get(element);
|
|
1418
1322
|
if (declarationList) {
|
|
1419
1323
|
this.styles.delete(element);
|
|
1420
|
-
element.setAttribute("style", generateStylesheetContent(declarationList, this.options));
|
|
1324
|
+
element.setAttribute("style", this.processorHelper.generateStylesheetContent(declarationList, this.options));
|
|
1421
1325
|
} else {
|
|
1422
1326
|
element.setAttribute("style", "");
|
|
1423
1327
|
}
|
|
@@ -1425,7 +1329,8 @@ class Processor {
|
|
|
1425
1329
|
}
|
|
1426
1330
|
|
|
1427
1331
|
insertVariables() {
|
|
1428
|
-
|
|
1332
|
+
const { cssVariables } = this.resources;
|
|
1333
|
+
if (cssVariables.size) {
|
|
1429
1334
|
const styleElement = this.doc.createElement("style");
|
|
1430
1335
|
const firstStyleElement = this.doc.head.querySelector("style");
|
|
1431
1336
|
if (firstStyleElement) {
|
|
@@ -1434,8 +1339,8 @@ class Processor {
|
|
|
1434
1339
|
this.doc.head.appendChild(styleElement);
|
|
1435
1340
|
}
|
|
1436
1341
|
let stylesheetContent = "";
|
|
1437
|
-
|
|
1438
|
-
|
|
1342
|
+
cssVariables.forEach(({ content, url }, indexResource) => {
|
|
1343
|
+
cssVariables.delete(indexResource);
|
|
1439
1344
|
if (stylesheetContent) {
|
|
1440
1345
|
stylesheetContent += ";";
|
|
1441
1346
|
}
|
|
@@ -1528,559 +1433,17 @@ class Processor {
|
|
|
1528
1433
|
}
|
|
1529
1434
|
}
|
|
1530
1435
|
|
|
1531
|
-
// ---------------
|
|
1532
|
-
// ProcessorHelper
|
|
1533
|
-
// ---------------
|
|
1534
|
-
const DATA_URI_PREFIX = "data:";
|
|
1535
|
-
const ABOUT_BLANK_URI = "about:blank";
|
|
1536
|
-
const REGEXP_URL_HASH = /(#.+?)$/;
|
|
1537
|
-
const SINGLE_FILE_VARIABLE_NAME_PREFIX = "--sf-img-";
|
|
1538
|
-
const SINGLE_FILE_VARIABLE_MAX_SIZE = 512 * 1024;
|
|
1539
|
-
|
|
1540
|
-
class ProcessorHelper {
|
|
1541
|
-
static setBackgroundImage(element, url, style) {
|
|
1542
|
-
element.style.setProperty("background-blend-mode", "normal", "important");
|
|
1543
|
-
element.style.setProperty("background-clip", "content-box", "important");
|
|
1544
|
-
element.style.setProperty("background-position", style && style["background-position"] ? style["background-position"] : "center", "important");
|
|
1545
|
-
element.style.setProperty("background-color", style && style["background-color"] ? style["background-color"] : "transparent", "important");
|
|
1546
|
-
element.style.setProperty("background-image", url, "important");
|
|
1547
|
-
element.style.setProperty("background-size", style && style["background-size"] ? style["background-size"] : "100% 100%", "important");
|
|
1548
|
-
element.style.setProperty("background-origin", "content-box", "important");
|
|
1549
|
-
element.style.setProperty("background-repeat", "no-repeat", "important");
|
|
1550
|
-
}
|
|
1551
|
-
|
|
1552
|
-
static processShortcutIcons(doc) {
|
|
1553
|
-
let shortcutIcon = findShortcutIcon(Array.from(doc.querySelectorAll("link[href][rel=\"shortcut icon\"]")));
|
|
1554
|
-
if (!shortcutIcon) {
|
|
1555
|
-
shortcutIcon = findShortcutIcon(Array.from(doc.querySelectorAll("link[href][rel=\"icon\"]")));
|
|
1556
|
-
}
|
|
1557
|
-
if (!shortcutIcon) {
|
|
1558
|
-
shortcutIcon = findShortcutIcon(Array.from(doc.querySelectorAll("link[href][rel*=\"icon\"]")));
|
|
1559
|
-
if (shortcutIcon) {
|
|
1560
|
-
shortcutIcon.rel = "shortcut icon";
|
|
1561
|
-
}
|
|
1562
|
-
}
|
|
1563
|
-
if (shortcutIcon) {
|
|
1564
|
-
doc.querySelectorAll("link[href][rel*=\"icon\"]").forEach(linkElement => {
|
|
1565
|
-
if (linkElement != shortcutIcon) {
|
|
1566
|
-
linkElement.remove();
|
|
1567
|
-
}
|
|
1568
|
-
});
|
|
1569
|
-
}
|
|
1570
|
-
}
|
|
1571
|
-
|
|
1572
|
-
static removeSingleLineCssComments(stylesheet) {
|
|
1573
|
-
if (stylesheet.children) {
|
|
1574
|
-
const removedRules = [];
|
|
1575
|
-
for (let cssRule = stylesheet.children.head; cssRule; cssRule = cssRule.next) {
|
|
1576
|
-
const ruleData = cssRule.data;
|
|
1577
|
-
if (ruleData.type == "Raw" && ruleData.value && ruleData.value.trim().startsWith("//")) {
|
|
1578
|
-
removedRules.push(cssRule);
|
|
1579
|
-
}
|
|
1580
|
-
}
|
|
1581
|
-
removedRules.forEach(cssRule => stylesheet.children.remove(cssRule));
|
|
1582
|
-
}
|
|
1583
|
-
}
|
|
1584
|
-
|
|
1585
|
-
static replacePseudoClassDefined(stylesheet) {
|
|
1586
|
-
const removedSelectors = [];
|
|
1587
|
-
if (stylesheet.children) {
|
|
1588
|
-
for (let cssRule = stylesheet.children.head; cssRule; cssRule = cssRule.next) {
|
|
1589
|
-
const ruleData = cssRule.data;
|
|
1590
|
-
if (ruleData.type == "Rule" && ruleData.prelude && ruleData.prelude.children) {
|
|
1591
|
-
for (let selector = ruleData.prelude.children.head; selector; selector = selector.next) {
|
|
1592
|
-
replacePseudoDefinedSelector(selector, ruleData.prelude);
|
|
1593
|
-
}
|
|
1594
|
-
}
|
|
1595
|
-
}
|
|
1596
|
-
}
|
|
1597
|
-
if (removedSelectors.length) {
|
|
1598
|
-
removedSelectors.forEach(({ parentSelector, selector }) => {
|
|
1599
|
-
if (parentSelector.data.children.size == 0 || !selector.prev || selector.prev.data.type == "Combinator" || selector.prev.data.type == "WhiteSpace") {
|
|
1600
|
-
parentSelector.data.children.replace(selector, cssTree.parse("*", { context: "selector" }).children.head);
|
|
1601
|
-
} else {
|
|
1602
|
-
parentSelector.data.children.remove(selector);
|
|
1603
|
-
}
|
|
1604
|
-
});
|
|
1605
|
-
}
|
|
1606
|
-
|
|
1607
|
-
function replacePseudoDefinedSelector(selector, parentSelector) {
|
|
1608
|
-
if (selector.data.children) {
|
|
1609
|
-
for (let childSelector = selector.data.children.head; childSelector; childSelector = childSelector.next) {
|
|
1610
|
-
replacePseudoDefinedSelector(childSelector, selector);
|
|
1611
|
-
}
|
|
1612
|
-
}
|
|
1613
|
-
if (selector.data.type == "PseudoClassSelector" && selector.data.name == "defined") {
|
|
1614
|
-
removedSelectors.push({ parentSelector, selector });
|
|
1615
|
-
}
|
|
1616
|
-
}
|
|
1617
|
-
}
|
|
1618
|
-
|
|
1619
|
-
static async resolveImportURLs(stylesheet, baseURI, options, workStylesheet, importedStyleSheets = new Set()) {
|
|
1620
|
-
let importFound;
|
|
1621
|
-
ProcessorHelper.resolveStylesheetURLs(stylesheet, baseURI, workStylesheet);
|
|
1622
|
-
const imports = getImportFunctions(stylesheet);
|
|
1623
|
-
await Promise.all(imports.map(async node => {
|
|
1624
|
-
const urlNode = cssTree.find(node, node => node.type == "Url") || cssTree.find(node, node => node.type == "String");
|
|
1625
|
-
if (urlNode) {
|
|
1626
|
-
let resourceURL = normalizeURL(urlNode.value);
|
|
1627
|
-
if (!testIgnoredPath(resourceURL) && testValidPath(resourceURL)) {
|
|
1628
|
-
urlNode.value = util.EMPTY_RESOURCE;
|
|
1629
|
-
try {
|
|
1630
|
-
resourceURL = util.resolveURL(resourceURL, baseURI);
|
|
1631
|
-
} catch (error) {
|
|
1632
|
-
// ignored
|
|
1633
|
-
}
|
|
1634
|
-
if (testValidURL(resourceURL) && !importedStyleSheets.has(resourceURL)) {
|
|
1635
|
-
const content = await getStylesheetContent(resourceURL);
|
|
1636
|
-
resourceURL = content.resourceURL;
|
|
1637
|
-
content.data = getUpdatedResourceContent(resourceURL, content, options);
|
|
1638
|
-
if (content.data && content.data.match(/^<!doctype /i)) {
|
|
1639
|
-
content.data = "";
|
|
1640
|
-
}
|
|
1641
|
-
const mediaQueryListNode = cssTree.find(node, node => node.type == "MediaQueryList");
|
|
1642
|
-
if (mediaQueryListNode) {
|
|
1643
|
-
content.data = wrapMediaQuery(content.data, cssTree.generate(mediaQueryListNode));
|
|
1644
|
-
}
|
|
1645
|
-
|
|
1646
|
-
content.data = content.data.replace(/:defined/gi, "*");
|
|
1647
|
-
|
|
1648
|
-
const importedStylesheet = cssTree.parse(content.data, { context: "stylesheet", parseCustomProperty: true });
|
|
1649
|
-
const ancestorStyleSheets = new Set(importedStyleSheets);
|
|
1650
|
-
ancestorStyleSheets.add(resourceURL);
|
|
1651
|
-
await ProcessorHelper.resolveImportURLs(importedStylesheet, resourceURL, options, workStylesheet, ancestorStyleSheets);
|
|
1652
|
-
for (let keyName of Object.keys(importedStylesheet)) {
|
|
1653
|
-
node[keyName] = importedStylesheet[keyName];
|
|
1654
|
-
}
|
|
1655
|
-
importFound = true;
|
|
1656
|
-
}
|
|
1657
|
-
}
|
|
1658
|
-
}
|
|
1659
|
-
}));
|
|
1660
|
-
return importFound;
|
|
1661
|
-
|
|
1662
|
-
async function getStylesheetContent(resourceURL) {
|
|
1663
|
-
const content = await util.getContent(resourceURL, {
|
|
1664
|
-
maxResourceSize: options.maxResourceSize,
|
|
1665
|
-
maxResourceSizeEnabled: options.maxResourceSizeEnabled,
|
|
1666
|
-
validateTextContentType: true,
|
|
1667
|
-
frameId: options.frameId,
|
|
1668
|
-
charset: options.charset,
|
|
1669
|
-
resourceReferrer: options.resourceReferrer,
|
|
1670
|
-
baseURI: options.baseURI,
|
|
1671
|
-
blockMixedContent: options.blockMixedContent,
|
|
1672
|
-
expectedType: "stylesheet",
|
|
1673
|
-
acceptHeaders: options.acceptHeaders,
|
|
1674
|
-
networkTimeout: options.networkTimeout
|
|
1675
|
-
});
|
|
1676
|
-
if (!(matchCharsetEquals(content.data, content.charset) || matchCharsetEquals(content.data, options.charset))) {
|
|
1677
|
-
options = Object.assign({}, options, { charset: getCharset(content.data) });
|
|
1678
|
-
return util.getContent(resourceURL, {
|
|
1679
|
-
maxResourceSize: options.maxResourceSize,
|
|
1680
|
-
maxResourceSizeEnabled: options.maxResourceSizeEnabled,
|
|
1681
|
-
validateTextContentType: true,
|
|
1682
|
-
frameId: options.frameId,
|
|
1683
|
-
charset: options.charset,
|
|
1684
|
-
resourceReferrer: options.resourceReferrer,
|
|
1685
|
-
baseURI: options.baseURI,
|
|
1686
|
-
blockMixedContent: options.blockMixedContent,
|
|
1687
|
-
expectedType: "stylesheet",
|
|
1688
|
-
acceptHeaders: options.acceptHeaders,
|
|
1689
|
-
networkTimeout: options.networkTimeout
|
|
1690
|
-
});
|
|
1691
|
-
} else {
|
|
1692
|
-
return content;
|
|
1693
|
-
}
|
|
1694
|
-
}
|
|
1695
|
-
}
|
|
1696
|
-
|
|
1697
|
-
static resolveStylesheetURLs(stylesheet, baseURI, workStylesheet) {
|
|
1698
|
-
const urls = getUrlFunctions(stylesheet);
|
|
1699
|
-
urls.map(urlNode => {
|
|
1700
|
-
const originalResourceURL = urlNode.value;
|
|
1701
|
-
let resourceURL = normalizeURL(originalResourceURL);
|
|
1702
|
-
if (!testIgnoredPath(resourceURL)) {
|
|
1703
|
-
workStylesheet.textContent = "tmp { content:\"" + resourceURL + "\"}";
|
|
1704
|
-
if (workStylesheet.sheet && workStylesheet.sheet.cssRules) {
|
|
1705
|
-
resourceURL = util.removeQuotes(workStylesheet.sheet.cssRules[0].style.getPropertyValue("content"));
|
|
1706
|
-
}
|
|
1707
|
-
if (!testIgnoredPath(resourceURL)) {
|
|
1708
|
-
if (!resourceURL || testValidPath(resourceURL)) {
|
|
1709
|
-
let resolvedURL;
|
|
1710
|
-
if (!originalResourceURL.startsWith("#")) {
|
|
1711
|
-
try {
|
|
1712
|
-
resolvedURL = util.resolveURL(resourceURL, baseURI);
|
|
1713
|
-
} catch (error) {
|
|
1714
|
-
// ignored
|
|
1715
|
-
}
|
|
1716
|
-
}
|
|
1717
|
-
if (testValidURL(resolvedURL)) {
|
|
1718
|
-
urlNode.value = resolvedURL;
|
|
1719
|
-
}
|
|
1720
|
-
} else {
|
|
1721
|
-
urlNode.value = util.EMPTY_RESOURCE;
|
|
1722
|
-
}
|
|
1723
|
-
}
|
|
1724
|
-
}
|
|
1725
|
-
});
|
|
1726
|
-
}
|
|
1727
|
-
|
|
1728
|
-
static async resolveLinkStylesheetURLs(resourceURL, baseURI, options, workStylesheet) {
|
|
1729
|
-
resourceURL = normalizeURL(resourceURL);
|
|
1730
|
-
if (resourceURL && resourceURL != baseURI && resourceURL != ABOUT_BLANK_URI) {
|
|
1731
|
-
const content = await util.getContent(resourceURL, {
|
|
1732
|
-
maxResourceSize: options.maxResourceSize,
|
|
1733
|
-
maxResourceSizeEnabled: options.maxResourceSizeEnabled,
|
|
1734
|
-
charset: options.charset,
|
|
1735
|
-
frameId: options.frameId,
|
|
1736
|
-
resourceReferrer: options.resourceReferrer,
|
|
1737
|
-
validateTextContentType: true,
|
|
1738
|
-
baseURI: baseURI,
|
|
1739
|
-
blockMixedContent: options.blockMixedContent,
|
|
1740
|
-
expectedType: "stylesheet",
|
|
1741
|
-
acceptHeaders: options.acceptHeaders,
|
|
1742
|
-
networkTimeout: options.networkTimeout
|
|
1743
|
-
});
|
|
1744
|
-
if (!(matchCharsetEquals(content.data, content.charset) || matchCharsetEquals(content.data, options.charset))) {
|
|
1745
|
-
options = Object.assign({}, options, { charset: getCharset(content.data) });
|
|
1746
|
-
return ProcessorHelper.resolveLinkStylesheetURLs(resourceURL, baseURI, options, workStylesheet);
|
|
1747
|
-
}
|
|
1748
|
-
resourceURL = content.resourceURL;
|
|
1749
|
-
content.data = getUpdatedResourceContent(content.resourceURL, content, options);
|
|
1750
|
-
if (content.data && content.data.match(/^<!doctype /i)) {
|
|
1751
|
-
content.data = "";
|
|
1752
|
-
}
|
|
1753
|
-
|
|
1754
|
-
content.data = content.data.replace(/:defined/gi, "*");
|
|
1755
|
-
|
|
1756
|
-
let stylesheet = cssTree.parse(content.data, { context: "stylesheet", parseCustomProperty: true });
|
|
1757
|
-
const importFound = await ProcessorHelper.resolveImportURLs(stylesheet, resourceURL, options, workStylesheet);
|
|
1758
|
-
if (importFound) {
|
|
1759
|
-
stylesheet = cssTree.parse(cssTree.generate(stylesheet), { context: "stylesheet", parseCustomProperty: true });
|
|
1760
|
-
}
|
|
1761
|
-
return stylesheet;
|
|
1762
|
-
}
|
|
1763
|
-
}
|
|
1764
|
-
|
|
1765
|
-
static async processStylesheet(cssRules, baseURI, options, cssVariables, batchRequest) {
|
|
1766
|
-
const promises = [];
|
|
1767
|
-
const removedRules = [];
|
|
1768
|
-
for (let cssRule = cssRules.head; cssRule; cssRule = cssRule.next) {
|
|
1769
|
-
const ruleData = cssRule.data;
|
|
1770
|
-
if (ruleData.type == "Atrule" && ruleData.name == "charset") {
|
|
1771
|
-
removedRules.push(cssRule);
|
|
1772
|
-
} else if (ruleData.block && ruleData.block.children) {
|
|
1773
|
-
if (ruleData.type == "Rule") {
|
|
1774
|
-
promises.push(this.processStyle(ruleData, baseURI, options, cssVariables, batchRequest));
|
|
1775
|
-
} else if (ruleData.type == "Atrule" && (ruleData.name == "media" || ruleData.name == "supports")) {
|
|
1776
|
-
promises.push(this.processStylesheet(ruleData.block.children, baseURI, options, cssVariables, batchRequest));
|
|
1777
|
-
} else if (ruleData.type == "Atrule" && ruleData.name == "font-face") {
|
|
1778
|
-
promises.push(processFontFaceRule(ruleData));
|
|
1779
|
-
}
|
|
1780
|
-
}
|
|
1781
|
-
}
|
|
1782
|
-
removedRules.forEach(cssRule => cssRules.remove(cssRule));
|
|
1783
|
-
await Promise.all(promises);
|
|
1784
|
-
|
|
1785
|
-
async function processFontFaceRule(ruleData) {
|
|
1786
|
-
const urls = getUrlFunctions(ruleData);
|
|
1787
|
-
await Promise.all(urls.map(async urlNode => {
|
|
1788
|
-
const originalResourceURL = urlNode.value;
|
|
1789
|
-
if (!options.blockFonts) {
|
|
1790
|
-
const resourceURL = normalizeURL(originalResourceURL);
|
|
1791
|
-
if (!testIgnoredPath(resourceURL) && testValidURL(resourceURL)) {
|
|
1792
|
-
let { content } = await batchRequest.addURL(resourceURL, { asBinary: true, expectedType: "font", baseURI, blockMixedContent: options.blockMixedContent });
|
|
1793
|
-
let resourceURLs = options.fontDeclarations.get(urlNode);
|
|
1794
|
-
if (!resourceURLs) {
|
|
1795
|
-
resourceURLs = [];
|
|
1796
|
-
options.fontDeclarations.set(urlNode, resourceURLs);
|
|
1797
|
-
}
|
|
1798
|
-
resourceURLs.push(resourceURL);
|
|
1799
|
-
if (!isDataURL(resourceURL) && options.saveOriginalURLs) {
|
|
1800
|
-
urlNode.value = "-sf-url-original(" + JSON.stringify(originalResourceURL) + ") " + content;
|
|
1801
|
-
} else {
|
|
1802
|
-
urlNode.value = content;
|
|
1803
|
-
}
|
|
1804
|
-
}
|
|
1805
|
-
} else {
|
|
1806
|
-
urlNode.value = util.EMPTY_RESOURCE;
|
|
1807
|
-
}
|
|
1808
|
-
}));
|
|
1809
|
-
}
|
|
1810
|
-
}
|
|
1811
|
-
|
|
1812
|
-
static async processStyle(ruleData, baseURI, options, cssVariables, batchRequest) {
|
|
1813
|
-
const urls = getUrlFunctions(ruleData);
|
|
1814
|
-
await Promise.all(urls.map(async urlNode => {
|
|
1815
|
-
const originalResourceURL = urlNode.value;
|
|
1816
|
-
if (!options.blockImages) {
|
|
1817
|
-
const resourceURL = normalizeURL(originalResourceURL);
|
|
1818
|
-
if (!testIgnoredPath(resourceURL) && testValidURL(resourceURL)) {
|
|
1819
|
-
let { content, indexResource, duplicate } = await batchRequest.addURL(resourceURL, { asBinary: true, expectedType: "image", groupDuplicates: options.groupDuplicateImages });
|
|
1820
|
-
if (!originalResourceURL.startsWith("#")) {
|
|
1821
|
-
const maxSizeDuplicateImages = options.maxSizeDuplicateImages || SINGLE_FILE_VARIABLE_MAX_SIZE;
|
|
1822
|
-
if (duplicate && options.groupDuplicateImages && util.getContentSize(content) < maxSizeDuplicateImages) {
|
|
1823
|
-
const varNode = cssTree.parse("var(" + SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource + ")", { context: "value" });
|
|
1824
|
-
for (let keyName of Object.keys(varNode.children.head.data)) {
|
|
1825
|
-
urlNode[keyName] = varNode.children.head.data[keyName];
|
|
1826
|
-
}
|
|
1827
|
-
cssVariables.set(indexResource, { content, url: originalResourceURL });
|
|
1828
|
-
} else {
|
|
1829
|
-
if (!isDataURL(resourceURL) && options.saveOriginalURLs) {
|
|
1830
|
-
urlNode.value = "-sf-url-original(" + JSON.stringify(originalResourceURL) + ") " + content;
|
|
1831
|
-
} else {
|
|
1832
|
-
urlNode.value = content;
|
|
1833
|
-
}
|
|
1834
|
-
}
|
|
1835
|
-
}
|
|
1836
|
-
}
|
|
1837
|
-
} else {
|
|
1838
|
-
urlNode.value = util.EMPTY_RESOURCE;
|
|
1839
|
-
}
|
|
1840
|
-
}));
|
|
1841
|
-
}
|
|
1842
|
-
|
|
1843
|
-
static async processAttribute(resourceElements, attributeName, baseURI, options, expectedType, cssVariables, styles, batchRequest, processDuplicates, removeElementIfMissing) {
|
|
1844
|
-
await Promise.all(Array.from(resourceElements).map(async resourceElement => {
|
|
1845
|
-
let resourceURL = resourceElement.getAttribute(attributeName);
|
|
1846
|
-
if (resourceURL != null) {
|
|
1847
|
-
resourceURL = normalizeURL(resourceURL);
|
|
1848
|
-
let originURL = resourceElement.dataset.singleFileOriginURL;
|
|
1849
|
-
if (options.saveOriginalURLs && !isDataURL(resourceURL)) {
|
|
1850
|
-
resourceElement.setAttribute("data-sf-original-" + attributeName, resourceURL);
|
|
1851
|
-
}
|
|
1852
|
-
delete resourceElement.dataset.singleFileOriginURL;
|
|
1853
|
-
if (!options["block" + expectedType.charAt(0).toUpperCase() + expectedType.substring(1) + "s"]) {
|
|
1854
|
-
if (!testIgnoredPath(resourceURL)) {
|
|
1855
|
-
setAttributeEmpty(resourceElement, attributeName, expectedType);
|
|
1856
|
-
if (testValidPath(resourceURL)) {
|
|
1857
|
-
try {
|
|
1858
|
-
resourceURL = util.resolveURL(resourceURL, baseURI);
|
|
1859
|
-
} catch (error) {
|
|
1860
|
-
// ignored
|
|
1861
|
-
}
|
|
1862
|
-
if (testValidURL(resourceURL)) {
|
|
1863
|
-
let { content, indexResource, duplicate } = await batchRequest.addURL(
|
|
1864
|
-
resourceURL,
|
|
1865
|
-
{ asBinary: true, expectedType, groupDuplicates: options.groupDuplicateImages && resourceElement.tagName.toUpperCase() == "IMG" && attributeName == "src" });
|
|
1866
|
-
if (originURL) {
|
|
1867
|
-
if (content == util.EMPTY_RESOURCE) {
|
|
1868
|
-
try {
|
|
1869
|
-
originURL = util.resolveURL(originURL, baseURI);
|
|
1870
|
-
} catch (error) {
|
|
1871
|
-
// ignored
|
|
1872
|
-
}
|
|
1873
|
-
try {
|
|
1874
|
-
resourceURL = originURL;
|
|
1875
|
-
content = (
|
|
1876
|
-
await util.getContent(resourceURL, {
|
|
1877
|
-
asBinary: true,
|
|
1878
|
-
expectedType,
|
|
1879
|
-
maxResourceSize: options.maxResourceSize,
|
|
1880
|
-
maxResourceSizeEnabled: options.maxResourceSizeEnabled,
|
|
1881
|
-
frameId: options.windowId,
|
|
1882
|
-
resourceReferrer: options.resourceReferrer,
|
|
1883
|
-
acceptHeaders: options.acceptHeaders,
|
|
1884
|
-
networkTimeout: options.networkTimeout
|
|
1885
|
-
})
|
|
1886
|
-
).data;
|
|
1887
|
-
} catch (error) {
|
|
1888
|
-
// ignored
|
|
1889
|
-
}
|
|
1890
|
-
}
|
|
1891
|
-
}
|
|
1892
|
-
if (removeElementIfMissing && content == util.EMPTY_RESOURCE) {
|
|
1893
|
-
resourceElement.remove();
|
|
1894
|
-
} else if (content !== util.EMPTY_RESOURCE) {
|
|
1895
|
-
let forbiddenPrefixFound = PREFIXES_FORBIDDEN_DATA_URI.filter(prefixDataURI => content.startsWith(prefixDataURI)).length;
|
|
1896
|
-
if (expectedType == "image") {
|
|
1897
|
-
if (forbiddenPrefixFound && Image) {
|
|
1898
|
-
forbiddenPrefixFound = await new Promise((resolve) => {
|
|
1899
|
-
const image = new Image();
|
|
1900
|
-
const timeoutId = setTimeout(() => resolve(true), 100);
|
|
1901
|
-
image.src = content;
|
|
1902
|
-
image.onload = () => cleanupAndResolve();
|
|
1903
|
-
image.onerror = () => cleanupAndResolve(true);
|
|
1904
|
-
|
|
1905
|
-
function cleanupAndResolve(value) {
|
|
1906
|
-
clearTimeout(timeoutId);
|
|
1907
|
-
resolve(value);
|
|
1908
|
-
}
|
|
1909
|
-
});
|
|
1910
|
-
}
|
|
1911
|
-
if (!forbiddenPrefixFound) {
|
|
1912
|
-
const isSVG = content.startsWith(PREFIX_DATA_URI_IMAGE_SVG);
|
|
1913
|
-
const maxSizeDuplicateImages = options.maxSizeDuplicateImages || SINGLE_FILE_VARIABLE_MAX_SIZE;
|
|
1914
|
-
if (processDuplicates && duplicate && !isSVG && util.getContentSize(content) < maxSizeDuplicateImages) {
|
|
1915
|
-
if (ProcessorHelper.replaceImageSource(resourceElement, SINGLE_FILE_VARIABLE_NAME_PREFIX + indexResource, options)) {
|
|
1916
|
-
cssVariables.set(indexResource, { content, url: originURL });
|
|
1917
|
-
const declarationList = cssTree.parse(resourceElement.getAttribute("style"), { context: "declarationList", parseCustomProperty: true });
|
|
1918
|
-
styles.set(resourceElement, declarationList);
|
|
1919
|
-
} else {
|
|
1920
|
-
resourceElement.setAttribute(attributeName, content);
|
|
1921
|
-
}
|
|
1922
|
-
} else {
|
|
1923
|
-
resourceElement.setAttribute(attributeName, content);
|
|
1924
|
-
}
|
|
1925
|
-
}
|
|
1926
|
-
} else {
|
|
1927
|
-
resourceElement.setAttribute(attributeName, content);
|
|
1928
|
-
}
|
|
1929
|
-
}
|
|
1930
|
-
}
|
|
1931
|
-
}
|
|
1932
|
-
}
|
|
1933
|
-
} else {
|
|
1934
|
-
setAttributeEmpty(resourceElement, attributeName, expectedType);
|
|
1935
|
-
}
|
|
1936
|
-
}
|
|
1937
|
-
}));
|
|
1938
|
-
|
|
1939
|
-
function setAttributeEmpty(resourceElement, attributeName, expectedType) {
|
|
1940
|
-
if (expectedType == "video" || expectedType == "audio") {
|
|
1941
|
-
resourceElement.removeAttribute(attributeName);
|
|
1942
|
-
} else {
|
|
1943
|
-
resourceElement.setAttribute(attributeName, util.EMPTY_RESOURCE);
|
|
1944
|
-
}
|
|
1945
|
-
}
|
|
1946
|
-
}
|
|
1947
|
-
|
|
1948
|
-
static async processXLinks(resourceElements, doc, baseURI, options, batchRequest) {
|
|
1949
|
-
let attributeName = "xlink:href";
|
|
1950
|
-
await Promise.all(Array.from(resourceElements).map(async resourceElement => {
|
|
1951
|
-
let originalResourceURL = resourceElement.getAttribute(attributeName);
|
|
1952
|
-
if (originalResourceURL == null) {
|
|
1953
|
-
attributeName = "href";
|
|
1954
|
-
originalResourceURL = resourceElement.getAttribute(attributeName);
|
|
1955
|
-
}
|
|
1956
|
-
if (options.saveOriginalURLs && !isDataURL(originalResourceURL)) {
|
|
1957
|
-
resourceElement.setAttribute("data-sf-original-href", originalResourceURL);
|
|
1958
|
-
}
|
|
1959
|
-
let resourceURL = normalizeURL(originalResourceURL);
|
|
1960
|
-
if (!options.blockImages) {
|
|
1961
|
-
if (testValidPath(resourceURL) && !testIgnoredPath(resourceURL)) {
|
|
1962
|
-
resourceElement.setAttribute(attributeName, util.EMPTY_RESOURCE);
|
|
1963
|
-
try {
|
|
1964
|
-
resourceURL = util.resolveURL(resourceURL, baseURI);
|
|
1965
|
-
} catch (error) {
|
|
1966
|
-
// ignored
|
|
1967
|
-
}
|
|
1968
|
-
if (testValidURL(resourceURL)) {
|
|
1969
|
-
const hashMatch = originalResourceURL.match(REGEXP_URL_HASH);
|
|
1970
|
-
if (originalResourceURL.startsWith(baseURI + "#")) {
|
|
1971
|
-
resourceElement.setAttribute(attributeName, hashMatch[0]);
|
|
1972
|
-
} else {
|
|
1973
|
-
const response = await batchRequest.addURL(resourceURL, { expectedType: "image" });
|
|
1974
|
-
const svgDoc = util.parseSVGContent(response.content);
|
|
1975
|
-
if (hashMatch && hashMatch[0]) {
|
|
1976
|
-
let symbolElement;
|
|
1977
|
-
try {
|
|
1978
|
-
symbolElement = svgDoc.querySelector(hashMatch[0]);
|
|
1979
|
-
} catch (error) {
|
|
1980
|
-
// ignored
|
|
1981
|
-
}
|
|
1982
|
-
if (symbolElement) {
|
|
1983
|
-
resourceElement.setAttribute(attributeName, hashMatch[0]);
|
|
1984
|
-
resourceElement.parentElement.insertBefore(symbolElement, resourceElement.parentElement.firstChild);
|
|
1985
|
-
}
|
|
1986
|
-
} else {
|
|
1987
|
-
const content = await batchRequest.addURL(resourceURL, { expectedType: "image" });
|
|
1988
|
-
resourceElement.setAttribute(attributeName, PREFIX_DATA_URI_IMAGE_SVG + "," + content);
|
|
1989
|
-
}
|
|
1990
|
-
}
|
|
1991
|
-
}
|
|
1992
|
-
} else if (resourceURL == options.url) {
|
|
1993
|
-
resourceElement.setAttribute(attributeName, originalResourceURL.substring(resourceURL.length));
|
|
1994
|
-
}
|
|
1995
|
-
} else {
|
|
1996
|
-
resourceElement.setAttribute(attributeName, util.EMPTY_RESOURCE);
|
|
1997
|
-
}
|
|
1998
|
-
}));
|
|
1999
|
-
}
|
|
2000
|
-
|
|
2001
|
-
static async processSrcset(resourceElements, baseURI, options, batchRequest) {
|
|
2002
|
-
await Promise.all(Array.from(resourceElements).map(async resourceElement => {
|
|
2003
|
-
const originSrcset = resourceElement.getAttribute("srcset");
|
|
2004
|
-
const srcset = util.parseSrcset(originSrcset);
|
|
2005
|
-
if (options.saveOriginalURLs && !isDataURL(originSrcset)) {
|
|
2006
|
-
resourceElement.setAttribute("data-sf-original-srcset", originSrcset);
|
|
2007
|
-
}
|
|
2008
|
-
if (!options.blockImages) {
|
|
2009
|
-
const srcsetValues = await Promise.all(srcset.map(async srcsetValue => {
|
|
2010
|
-
let resourceURL = normalizeURL(srcsetValue.url);
|
|
2011
|
-
if (!testIgnoredPath(resourceURL)) {
|
|
2012
|
-
if (testValidPath(resourceURL)) {
|
|
2013
|
-
try {
|
|
2014
|
-
resourceURL = util.resolveURL(resourceURL, baseURI);
|
|
2015
|
-
} catch (error) {
|
|
2016
|
-
// ignored
|
|
2017
|
-
}
|
|
2018
|
-
if (testValidURL(resourceURL)) {
|
|
2019
|
-
const { content } = await batchRequest.addURL(resourceURL, { asBinary: true, expectedType: "image" });
|
|
2020
|
-
const forbiddenPrefixFound = PREFIXES_FORBIDDEN_DATA_URI.filter(prefixDataURI => content.startsWith(prefixDataURI)).length;
|
|
2021
|
-
if (forbiddenPrefixFound) {
|
|
2022
|
-
return "";
|
|
2023
|
-
}
|
|
2024
|
-
return content + (srcsetValue.w ? " " + srcsetValue.w + "w" : srcsetValue.d ? " " + srcsetValue.d + "x" : "");
|
|
2025
|
-
} else {
|
|
2026
|
-
return "";
|
|
2027
|
-
}
|
|
2028
|
-
} else {
|
|
2029
|
-
return "";
|
|
2030
|
-
}
|
|
2031
|
-
} else {
|
|
2032
|
-
return resourceURL + (srcsetValue.w ? " " + srcsetValue.w + "w" : srcsetValue.d ? " " + srcsetValue.d + "x" : "");
|
|
2033
|
-
}
|
|
2034
|
-
}));
|
|
2035
|
-
resourceElement.setAttribute("srcset", srcsetValues.join(", "));
|
|
2036
|
-
} else {
|
|
2037
|
-
resourceElement.setAttribute("srcset", "");
|
|
2038
|
-
}
|
|
2039
|
-
}));
|
|
2040
|
-
}
|
|
2041
|
-
|
|
2042
|
-
static replaceImageSource(imgElement, variableName, options) {
|
|
2043
|
-
const attributeValue = imgElement.getAttribute(util.IMAGE_ATTRIBUTE_NAME);
|
|
2044
|
-
if (attributeValue) {
|
|
2045
|
-
const imageData = options.images[Number(imgElement.getAttribute(util.IMAGE_ATTRIBUTE_NAME))];
|
|
2046
|
-
if (imageData && imageData.replaceable) {
|
|
2047
|
-
imgElement.setAttribute("src", `${PREFIX_DATA_URI_IMAGE_SVG},<svg xmlns="http://www.w3.org/2000/svg" width="${imageData.size.pxWidth}" height="${imageData.size.pxHeight}"><rect fill-opacity="0"/></svg>`);
|
|
2048
|
-
const backgroundStyle = {};
|
|
2049
|
-
const backgroundSize = (imageData.objectFit == "content" || imageData.objectFit == "cover") && imageData.objectFit;
|
|
2050
|
-
if (backgroundSize) {
|
|
2051
|
-
backgroundStyle["background-size"] = imageData.objectFit;
|
|
2052
|
-
}
|
|
2053
|
-
if (imageData.objectPosition) {
|
|
2054
|
-
backgroundStyle["background-position"] = imageData.objectPosition;
|
|
2055
|
-
}
|
|
2056
|
-
if (imageData.backgroundColor) {
|
|
2057
|
-
backgroundStyle["background-color"] = imageData.backgroundColor;
|
|
2058
|
-
}
|
|
2059
|
-
ProcessorHelper.setBackgroundImage(imgElement, "var(" + variableName + ")", backgroundStyle);
|
|
2060
|
-
imgElement.removeAttribute(util.IMAGE_ATTRIBUTE_NAME);
|
|
2061
|
-
return true;
|
|
2062
|
-
}
|
|
2063
|
-
}
|
|
2064
|
-
}
|
|
2065
|
-
}
|
|
2066
|
-
|
|
2067
1436
|
// ----
|
|
2068
1437
|
// Util
|
|
2069
1438
|
// ----
|
|
1439
|
+
const DATA_URI_PREFIX = "data:";
|
|
1440
|
+
const ABOUT_BLANK_URI = "about:blank";
|
|
2070
1441
|
const BLOB_URI_PREFIX = "blob:";
|
|
2071
1442
|
const HTTP_URI_PREFIX = /^https?:\/\//;
|
|
2072
1443
|
const FILE_URI_PREFIX = /^file:\/\//;
|
|
2073
1444
|
const EMPTY_URL = /^https?:\/\/+\s*$/;
|
|
2074
1445
|
const NOT_EMPTY_URL = /^(https?:\/\/|file:\/\/|blob:).+/;
|
|
2075
|
-
|
|
2076
|
-
function getUpdatedResourceContent(resourceURL, content, options) {
|
|
2077
|
-
if (options.rootDocument && options.updatedResources[resourceURL]) {
|
|
2078
|
-
options.updatedResources[resourceURL].retrieved = true;
|
|
2079
|
-
return options.updatedResources[resourceURL].content;
|
|
2080
|
-
} else {
|
|
2081
|
-
return content.data || "";
|
|
2082
|
-
}
|
|
2083
|
-
}
|
|
1446
|
+
const SINGLE_FILE_VARIABLE_NAME_PREFIX = "--sf-img-";
|
|
2084
1447
|
|
|
2085
1448
|
function normalizeURL(url) {
|
|
2086
1449
|
if (!url || url.startsWith(DATA_URI_PREFIX)) {
|
|
@@ -2090,22 +1453,6 @@ function normalizeURL(url) {
|
|
|
2090
1453
|
}
|
|
2091
1454
|
}
|
|
2092
1455
|
|
|
2093
|
-
function matchCharsetEquals(stylesheetContent = "", charset = UTF8_CHARSET) {
|
|
2094
|
-
const stylesheetCharset = getCharset(stylesheetContent);
|
|
2095
|
-
if (stylesheetCharset) {
|
|
2096
|
-
return stylesheetCharset == charset.toLowerCase();
|
|
2097
|
-
} else {
|
|
2098
|
-
return true;
|
|
2099
|
-
}
|
|
2100
|
-
}
|
|
2101
|
-
|
|
2102
|
-
function getCharset(stylesheetContent = "") {
|
|
2103
|
-
const match = stylesheetContent.match(/^@charset\s+"([^"]*)";/i);
|
|
2104
|
-
if (match && match[1]) {
|
|
2105
|
-
return match[1].toLowerCase().trim();
|
|
2106
|
-
}
|
|
2107
|
-
}
|
|
2108
|
-
|
|
2109
1456
|
function getOnEventAttributeNames(doc) {
|
|
2110
1457
|
const element = doc.createElement("div");
|
|
2111
1458
|
const attributeNames = [];
|
|
@@ -2118,39 +1465,10 @@ function getOnEventAttributeNames(doc) {
|
|
|
2118
1465
|
return attributeNames;
|
|
2119
1466
|
}
|
|
2120
1467
|
|
|
2121
|
-
function getUrlFunctions(declarationList) {
|
|
2122
|
-
return cssTree.findAll(declarationList, node => node.type == "Url");
|
|
2123
|
-
}
|
|
2124
|
-
|
|
2125
|
-
function getImportFunctions(declarationList) {
|
|
2126
|
-
return cssTree.findAll(declarationList, node => node.type == "Atrule" && node.name == "import");
|
|
2127
|
-
}
|
|
2128
|
-
|
|
2129
|
-
function generateStylesheetContent(stylesheet, options) {
|
|
2130
|
-
let stylesheetContent = cssTree.generate(stylesheet);
|
|
2131
|
-
if (options.compressCSS) {
|
|
2132
|
-
stylesheetContent = util.compressCSS(stylesheetContent);
|
|
2133
|
-
}
|
|
2134
|
-
if (options.saveOriginalURLs) {
|
|
2135
|
-
stylesheetContent = replaceOriginalURLs(stylesheetContent);
|
|
2136
|
-
}
|
|
2137
|
-
return stylesheetContent;
|
|
2138
|
-
}
|
|
2139
|
-
|
|
2140
|
-
function findShortcutIcon(shortcutIcons) {
|
|
2141
|
-
shortcutIcons = shortcutIcons.filter(linkElement => linkElement.href != util.EMPTY_RESOURCE);
|
|
2142
|
-
shortcutIcons.sort((linkElement1, linkElement2) => (parseInt(linkElement2.sizes, 10) || 16) - (parseInt(linkElement1.sizes, 10) || 16));
|
|
2143
|
-
return shortcutIcons[0];
|
|
2144
|
-
}
|
|
2145
|
-
|
|
2146
1468
|
function isDataURL(url) {
|
|
2147
1469
|
return url && (url.startsWith(DATA_URI_PREFIX) || url.startsWith(BLOB_URI_PREFIX));
|
|
2148
1470
|
}
|
|
2149
1471
|
|
|
2150
|
-
function replaceOriginalURLs(stylesheetContent) {
|
|
2151
|
-
return stylesheetContent.replace(/url\(-sf-url-original\\\(\\"(.*?)\\"\\\)\\ /g, "/* original URL: $1 */url(");
|
|
2152
|
-
}
|
|
2153
|
-
|
|
2154
1472
|
function testIgnoredPath(resourceURL) {
|
|
2155
1473
|
return resourceURL && (resourceURL.startsWith(DATA_URI_PREFIX) || resourceURL == ABOUT_BLANK_URI);
|
|
2156
1474
|
}
|
|
@@ -2163,14 +1481,6 @@ function testValidURL(resourceURL) {
|
|
|
2163
1481
|
return testValidPath(resourceURL) && (resourceURL.match(HTTP_URI_PREFIX) || resourceURL.match(FILE_URI_PREFIX) || resourceURL.startsWith(BLOB_URI_PREFIX)) && resourceURL.match(NOT_EMPTY_URL);
|
|
2164
1482
|
}
|
|
2165
1483
|
|
|
2166
|
-
function wrapMediaQuery(stylesheetContent, mediaQuery) {
|
|
2167
|
-
if (mediaQuery) {
|
|
2168
|
-
return "@media " + mediaQuery + "{ " + stylesheetContent + " }";
|
|
2169
|
-
} else {
|
|
2170
|
-
return stylesheetContent;
|
|
2171
|
-
}
|
|
2172
|
-
}
|
|
2173
|
-
|
|
2174
1484
|
function log(...args) {
|
|
2175
1485
|
console.log("S-File <core> ", ...args); // eslint-disable-line no-console
|
|
2176
1486
|
}
|