single-file-cli 2.9.2 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.sh +1 -1
- package/deno.json +1 -1
- package/lib/bidi-client.js +3 -4
- package/lib/cdp-client.js +1 -1
- package/lib/single-file-archive.js +6 -5
- package/lib/single-file-bundle.js +1 -1
- package/lib/single-file-script.js +6 -5
- package/lib/version.js +1 -1
- package/options.js +5 -2
- package/package.json +1 -1
- package/single-file-cli-api.js +42 -22
- package/test/e2e/browser-script.test.js +40 -0
- package/test/e2e/canvas-round-trip.test.js +5 -1
- package/test/e2e/dump-json.test.js +111 -0
- package/test/e2e/font-face-source-order.test.js +137 -0
- package/test/e2e/html-at-media-url.test.js +3 -1
- package/test/e2e/insert-head-elements.test.js +71 -0
- package/test/e2e/max-appended-data-length.test.js +100 -0
- package/test/unit/archive-options.test.js +51 -0
|
@@ -93,14 +93,15 @@ function initSingleFile(constants) {
|
|
|
93
93
|
});
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
-
async function getScriptSource(options) {
|
|
96
|
+
async function getScriptSource(options, globalAssignment = "") {
|
|
97
97
|
let source = "";
|
|
98
98
|
source += script;
|
|
99
|
-
source +=
|
|
99
|
+
source += "\n" + globalAssignment;
|
|
100
|
+
source += "\n" + await readScriptFiles(options && options.browserScripts ? options.browserScripts : []);
|
|
100
101
|
if (options.browserStylesheets && options.browserStylesheets.length) {
|
|
101
|
-
source += "
|
|
102
|
+
source += "\naddEventListener(\"load\",()=>{const styleElement=document.createElement(\"style\");styleElement.textContent=" + JSON.stringify(await readScriptFiles(options.browserStylesheets)) + ";document.body.appendChild(styleElement);});";
|
|
102
103
|
}
|
|
103
|
-
source += "(" + initSingleFile.toString() + ")(" + JSON.stringify({
|
|
104
|
+
source += "\n(" + initSingleFile.toString() + ")(" + JSON.stringify({
|
|
104
105
|
FETCH_FUNCTION_NAME,
|
|
105
106
|
RESOLVE_FETCH_FUNCTION_NAME,
|
|
106
107
|
REJECT_FETCH_FUNCTION_NAME
|
|
@@ -161,5 +162,5 @@ function getPageDataScriptSource(options, [SET_SCREENSHOT_FUNCTION_NAME, SET_PDF
|
|
|
161
162
|
|
|
162
163
|
|
|
163
164
|
async function readScriptFiles(paths) {
|
|
164
|
-
return (await Promise.all(paths.map(path => readTextFile(path)))).join("");
|
|
165
|
+
return (await Promise.all(paths.map(path => readTextFile(path)))).join("\n");
|
|
165
166
|
}
|
package/lib/version.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const version = "2.
|
|
1
|
+
export const version = "2.11.0";
|
package/options.js
CHANGED
|
@@ -104,7 +104,7 @@ const OPTIONS_INFO = [{
|
|
|
104
104
|
"block-scripts": { description: "Block scripts", type: "boolean", defaultValue: true },
|
|
105
105
|
"block-stylesheets": { description: "Block stylesheets", type: "boolean", defaultValue: false },
|
|
106
106
|
"block-videos": { description: "Block videos", type: "boolean", defaultValue: true },
|
|
107
|
-
"block-mixed-content": { description: "Block
|
|
107
|
+
"block-mixed-content": { description: "Block active content (i.e. stylesheets, scripts, fonts) served from HTTP in HTTPS pages, like browsers do; images, videos and audios are unaffected", type: "boolean" },
|
|
108
108
|
"blocked-url-pattern": { key: "blockedURLPatterns", description: "Regular expression matching URLs to block (e.g. 'annoying-banners\\.com')", type: "string[]" }
|
|
109
109
|
}, {
|
|
110
110
|
"load-deferred-images": { description: "Load deferred (a.k.a. lazy-loaded) images", type: "boolean", defaultValue: true },
|
|
@@ -113,7 +113,7 @@ const OPTIONS_INFO = [{
|
|
|
113
113
|
"load-deferred-images-keep-zoom-level": { description: "Load deferred images by keeping zoomed out the page", type: "boolean" },
|
|
114
114
|
"load-deferred-images-before-frames": { description: "Load deferred frames before before saving fame contents", type: "boolean" },
|
|
115
115
|
"max-resource-size-enabled": { description: "Enable removal of embedded resources exceeding a given size", type: "boolean" },
|
|
116
|
-
"max-resource-size": { description: "Maximum size of embedded resources in MB
|
|
116
|
+
"max-resource-size": { description: "Maximum size of embedded resources in MB. It applies to every fetched resource, i.e. images, fonts, stylesheets, scripts, frames, videos and audios; a resource above the limit is left out of the saved page", type: "number", defaultValue: 10 }
|
|
117
117
|
}, {
|
|
118
118
|
"compress-css": { key: "compressCSS", description: "Compress CSS stylesheets", type: "boolean" },
|
|
119
119
|
"compress-html": { key: "compressHTML", description: "Compress HTML content", type: "boolean", defaultValue: true },
|
|
@@ -141,6 +141,7 @@ const OPTIONS_INFO = [{
|
|
|
141
141
|
"extract-data-from-page": { description: "Extract compressed data from the page instead of fetching the page in order to create universal self-extracting HTML files", type: "boolean", defaultValue: true },
|
|
142
142
|
"prevent-appended-data": { description: "Prevent appending data after the compressed data when creating self-extracting HTML files", type: "boolean" },
|
|
143
143
|
"declare-appended-data": { description: "Declare the data appended after the compressed data as the comment of the ZIP archive, for readers rejecting undeclared trailing bytes (e.g. java.util.zip); ZIP tools then print that data when listing the archive. Has no effect when the extra data is relocated ahead of the compressed data, because nothing is appended then", type: "boolean" },
|
|
144
|
+
"max-appended-data-length": { description: "Maximum number of bytes appended after the compressed data when creating self-extracting HTML files, 16361 by default. Readers only tolerate trailing bytes as far back as their end-of-archive scan reaches (16383 bytes for libarchive, 32768 for perl Archive::Zip, 65557 for Python zipfile); the default fits the narrowest. Data over the limit is relocated ahead of the compressed data instead. Raising the limit above 65535 silently defeats --declare-appended-data: the declaration is the length of the ZIP comment, a 16-bit field, so a longer run is appended undeclared and without a warning", type: "number" },
|
|
144
145
|
"embed-screenshot": { description: "Embed a screenshot of the page as a PNG file in the compressed file (self-extracting HTML or ZIP file). When enabled, the resulting file can be read as a ZIP file or a PNG image.", type: "boolean" },
|
|
145
146
|
"embed-screenshot-options": { description: "Options passed to the CDP method `Page.captureScreenshot()` given as a JSON string (e.g. { \"captureBeyondViewport\": false })", type: "string" },
|
|
146
147
|
"embedded-image": { description: "Path to a PNG image to embed in the compressed file. Unlike --embed-screenshot it is also compatible with --crawl-save-archive, where it becomes the image of the whole archive.", type: "string" },
|
|
@@ -192,6 +193,7 @@ const OPTIONS_INFO = [{
|
|
|
192
193
|
}, {
|
|
193
194
|
"include-bom": { key: "includeBOM", description: "Include the UTF-8 BOM into the HTML page, ignored when the page is compressed unless --extract-data-from-page is disabled and no image is embedded", type: "boolean" },
|
|
194
195
|
"insert-meta-csp": { key: "insertMetaCSP", description: "Include a <meta> tag with a CSP to avoid potential requests to internet when viewing a page", type: "boolean", defaultValue: true },
|
|
196
|
+
"insert-meta-noindex": { key: "insertMetaNoIndex", description: "Insert a <meta name=robots content=noindex> element, so a saved page served on a public host is not indexed. Ignored when the page already declares noindex", type: "boolean" },
|
|
195
197
|
"remove-saved-date": { description: "Remove saved date metadata in HTML header", type: "boolean" },
|
|
196
198
|
"save-original-urls": { key: "saveOriginalURLs", description: "Save the original URLS in the embedded contents", type: "boolean" },
|
|
197
199
|
"insert-single-file-comment": { description: "Insert a comment in the HTML header with the URL of the page", type: "boolean", defaultValue: true },
|
|
@@ -211,6 +213,7 @@ const OPTIONS_INFO = [{
|
|
|
211
213
|
|
|
212
214
|
"save-raw-page": { description: "Save the original page without interpreting it into the browser", type: "boolean" },
|
|
213
215
|
"output-json": { description: "Output the result as a JSON string containing the page and network info", type: "boolean" },
|
|
216
|
+
"dump-json": { description: "Write the same JSON as --output-json to stdout, without the page content, and leave the page file alone. Unlike --output-json the page is still written where it would have been, so the two outputs can be combined. Incompatible with --output-json, and with --dump-content unless --output is set, because both write to stdout", type: "boolean" },
|
|
214
217
|
"create-browser-profile": { description: "Path of the browser profile directory to create or update instead of saving a page. The browser is started with a visible window on the URL passed as argument, log in to the website and quit the browser to save the profile, then pass it to --browser-profile when saving pages", type: "string" },
|
|
215
218
|
|
|
216
219
|
}, {
|
package/package.json
CHANGED
package/single-file-cli-api.js
CHANGED
|
@@ -27,11 +27,13 @@ import { Buffer } from "node:buffer";
|
|
|
27
27
|
import * as cdpBackend from "./lib/cdp-client.js";
|
|
28
28
|
import * as bidiBackend from "./lib/bidi-client.js";
|
|
29
29
|
import { getZipScriptSource } from "./lib/single-file-script.js";
|
|
30
|
-
import { createPagesArchive } from "./lib/single-file-archive.js";
|
|
30
|
+
import { createPagesArchive, PROCESS_OPTION_NAMES } from "./lib/single-file-archive.js";
|
|
31
31
|
import { Deno, path } from "./lib/deno-polyfill.js";
|
|
32
32
|
|
|
33
33
|
const VALID_URL_TEST = /^(https?|file):\/\//;
|
|
34
34
|
|
|
35
|
+
const ARCHIVE_EXCLUDED_OPTION_NAMES = ["createRootDirectory", "disableCompression", "insertTextBody", "password", "url"];
|
|
36
|
+
|
|
35
37
|
const DEFAULT_OPTIONS = {
|
|
36
38
|
removeHiddenElements: true,
|
|
37
39
|
removeUnusedStyles: true,
|
|
@@ -64,7 +66,7 @@ const STATE_PROCESSED = "processed";
|
|
|
64
66
|
const { readTextFile, writeTextFile, readFile, writeFile, stdout, mkdir, makeTempDir, remove, stat, errors } = Deno;
|
|
65
67
|
let backend = cdpBackend, tasks = [], maxParallelWorkers, sessionFilename, archiveTempDirectory, errorCount = 0;
|
|
66
68
|
|
|
67
|
-
export { initialize, closeBrowser };
|
|
69
|
+
export { initialize, closeBrowser, getArchiveOptions, ARCHIVE_EXCLUDED_OPTION_NAMES };
|
|
68
70
|
|
|
69
71
|
async function closeBrowser() {
|
|
70
72
|
await backend.closeBrowser();
|
|
@@ -75,6 +77,12 @@ async function initialize(options) {
|
|
|
75
77
|
if ((options.embedPdf || options.embeddedPdf || options.embedScreenshot || options.embeddedImage) && !options.compressContent) {
|
|
76
78
|
throw new Error("--embed-pdf, --embedded-pdf, --embed-screenshot and --embedded-image require --compress-content");
|
|
77
79
|
}
|
|
80
|
+
if (options.dumpJson && options.outputJson) {
|
|
81
|
+
throw new Error("--dump-json is not compatible with --output-json, which already writes the JSON in place of the page");
|
|
82
|
+
}
|
|
83
|
+
if (options.dumpJson && options.dumpContent && !options.output) {
|
|
84
|
+
throw new Error("--dump-json is not compatible with --dump-content unless --output is set, because both write to stdout");
|
|
85
|
+
}
|
|
78
86
|
if (options.crawlSaveArchiveDedup && !options.crawlSaveArchive) {
|
|
79
87
|
throw new Error("--crawl-save-archive-dedup requires --crawl-save-archive");
|
|
80
88
|
}
|
|
@@ -196,6 +204,23 @@ async function finish(options) {
|
|
|
196
204
|
return errorCount;
|
|
197
205
|
}
|
|
198
206
|
|
|
207
|
+
function getArchiveOptions(options) {
|
|
208
|
+
const archiveOptions = {
|
|
209
|
+
zipScript: getZipScriptSource(),
|
|
210
|
+
dedupPages: options.crawlSaveArchiveDedup,
|
|
211
|
+
markUnarchivedLinks: options.crawlSaveArchiveMarkUnarchivedLinks,
|
|
212
|
+
tocPage: options.crawlSaveArchiveToc,
|
|
213
|
+
pageList: options.crawlSaveArchivePageList,
|
|
214
|
+
pageTransitions: options.crawlSaveArchivePageTransitions,
|
|
215
|
+
insertSingleFileComment: options.insertSingleFileComment,
|
|
216
|
+
removeSavedDate: options.removeSavedDate
|
|
217
|
+
};
|
|
218
|
+
PROCESS_OPTION_NAMES
|
|
219
|
+
.filter(name => !ARCHIVE_EXCLUDED_OPTION_NAMES.includes(name) && !(name in archiveOptions))
|
|
220
|
+
.forEach(name => archiveOptions[name] = options[name]);
|
|
221
|
+
return archiveOptions;
|
|
222
|
+
}
|
|
223
|
+
|
|
199
224
|
async function savePagesArchive(options) {
|
|
200
225
|
const archiveTasks = tasks.filter(task => task.archiveFilename);
|
|
201
226
|
if (archiveTasks.length) {
|
|
@@ -205,26 +230,7 @@ async function savePagesArchive(options) {
|
|
|
205
230
|
title: task.title,
|
|
206
231
|
getData: () => readFile(task.archiveFilename)
|
|
207
232
|
}));
|
|
208
|
-
const content = await createPagesArchive(pages,
|
|
209
|
-
zipScript: getZipScriptSource(),
|
|
210
|
-
dedupPages: options.crawlSaveArchiveDedup,
|
|
211
|
-
markUnarchivedLinks: options.crawlSaveArchiveMarkUnarchivedLinks,
|
|
212
|
-
tocPage: options.crawlSaveArchiveToc,
|
|
213
|
-
pageList: options.crawlSaveArchivePageList,
|
|
214
|
-
pageTransitions: options.crawlSaveArchivePageTransitions,
|
|
215
|
-
selfExtractingArchive: options.selfExtractingArchive,
|
|
216
|
-
extractDataFromPage: options.extractDataFromPage,
|
|
217
|
-
preventAppendedData: options.preventAppendedData,
|
|
218
|
-
declareAppendedData: options.declareAppendedData,
|
|
219
|
-
embeddedPdf: options.embeddedPdf,
|
|
220
|
-
embeddedImage: options.embeddedImage,
|
|
221
|
-
includeBOM: options.includeBOM,
|
|
222
|
-
insertMetaCSP: options.insertMetaCSP,
|
|
223
|
-
insertCanonicalLink: options.insertCanonicalLink,
|
|
224
|
-
insertMetaNoIndex: options.insertMetaNoIndex,
|
|
225
|
-
insertSingleFileComment: options.insertSingleFileComment,
|
|
226
|
-
removeSavedDate: options.removeSavedDate
|
|
227
|
-
});
|
|
233
|
+
const content = await createPagesArchive(pages, getArchiveOptions(options));
|
|
228
234
|
if (options.dumpContent && !options.output) {
|
|
229
235
|
await stdout.write(content);
|
|
230
236
|
} else {
|
|
@@ -400,6 +406,7 @@ async function capturePage(options) {
|
|
|
400
406
|
if (options.archiveFilename) {
|
|
401
407
|
await writeFile(options.archiveFilename, content);
|
|
402
408
|
pageData.archiveFilename = options.archiveFilename;
|
|
409
|
+
dumpJsonMetadata(pageData, options);
|
|
403
410
|
return pageData;
|
|
404
411
|
}
|
|
405
412
|
if (options.outputJson) {
|
|
@@ -429,6 +436,7 @@ async function capturePage(options) {
|
|
|
429
436
|
const outputDirectory = getOutputDirectory(options);
|
|
430
437
|
pageData.filename = filename.startsWith(outputDirectory) ? filename.substring(outputDirectory.length) : filename;
|
|
431
438
|
}
|
|
439
|
+
dumpJsonMetadata(pageData, options);
|
|
432
440
|
return pageData;
|
|
433
441
|
} catch (error) {
|
|
434
442
|
errorCount++;
|
|
@@ -453,6 +461,18 @@ async function capturePage(options) {
|
|
|
453
461
|
}
|
|
454
462
|
}
|
|
455
463
|
|
|
464
|
+
function dumpJsonMetadata(pageData, options) {
|
|
465
|
+
if (options.dumpJson) {
|
|
466
|
+
const metadata = Object.assign({}, pageData);
|
|
467
|
+
delete metadata.content;
|
|
468
|
+
delete metadata.binaryContent;
|
|
469
|
+
delete metadata.doctype;
|
|
470
|
+
delete metadata.viewport;
|
|
471
|
+
delete metadata.comment;
|
|
472
|
+
console.log(JSON.stringify(metadata, null, 2)); // eslint-disable-line no-console
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
|
|
456
476
|
function getOutputDirectory(options) {
|
|
457
477
|
if (Array.isArray(options.outputDirectory)) {
|
|
458
478
|
const outputDirectory = options.outputDirectory.pop();
|
|
@@ -58,6 +58,46 @@ test("a browser script that throws fails the save instead of degrading it", { ti
|
|
|
58
58
|
});
|
|
59
59
|
});
|
|
60
60
|
|
|
61
|
+
// The same concatenation swallowed code across file boundaries. The scripts were joined with "",
|
|
62
|
+
// and the initSingleFile call appended straight onto the last one, so a file whose final line was a
|
|
63
|
+
// // comment commented out whatever came next. It only bites when that file has no trailing newline,
|
|
64
|
+
// which is why it went unnoticed: an editor that adds one hides it completely. The last-file case
|
|
65
|
+
// failed loudly (initSingleFile never ran, "no valid SingleFile execution context"), but a file
|
|
66
|
+
// eating the NEXT script was silent — exit 0, a valid save, and one script quietly skipped.
|
|
67
|
+
test("a browser script ending in a comment does not swallow the next one", { timeout: TEST_TIMEOUT }, async () => {
|
|
68
|
+
const MARKER = "second-script-marker";
|
|
69
|
+
await withPage(async ({ url, directory }) => {
|
|
70
|
+
const firstPath = join(directory, "first.js");
|
|
71
|
+
const secondPath = join(directory, "second.js");
|
|
72
|
+
// deliberately no trailing newline: that is the whole trigger
|
|
73
|
+
await writeFile(firstPath, "globalThis.__first = 1; // a trailing comment");
|
|
74
|
+
await writeFile(secondPath, "addEventListener(\"DOMContentLoaded\",()=>{" +
|
|
75
|
+
"const marker=document.createElement(\"p\");marker.id=\"" + MARKER + "\";document.body.appendChild(marker);});\n");
|
|
76
|
+
const outputPath = join(directory, "out.html");
|
|
77
|
+
await execFileAsync(process.execPath, [
|
|
78
|
+
"single-file-node.js", url, outputPath,
|
|
79
|
+
"--browser-script", firstPath,
|
|
80
|
+
"--browser-script", secondPath
|
|
81
|
+
], { cwd: cliDirectory });
|
|
82
|
+
assert.match(await readFile(outputPath, "utf8"), new RegExp(MARKER),
|
|
83
|
+
"the first script's trailing comment swallowed the second one");
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("a lone browser script ending in a comment still saves the page", { timeout: TEST_TIMEOUT }, async () => {
|
|
88
|
+
await withPage(async ({ url, directory }) => {
|
|
89
|
+
const scriptPath = join(directory, "only.js");
|
|
90
|
+
await writeFile(scriptPath, "globalThis.__only = 1; // a trailing comment");
|
|
91
|
+
const outputPath = join(directory, "out.html");
|
|
92
|
+
await execFileAsync(process.execPath, [
|
|
93
|
+
"single-file-node.js", url, outputPath,
|
|
94
|
+
"--browser-script", scriptPath
|
|
95
|
+
], { cwd: cliDirectory });
|
|
96
|
+
assert.match(await readFile(outputPath, "utf8"), /<h1>page<\/h1>/,
|
|
97
|
+
"the trailing comment reached the initSingleFile call");
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
|
|
61
101
|
async function withPage(run) {
|
|
62
102
|
const server = createServer((request, response) =>
|
|
63
103
|
response.writeHead(200, { "content-type": "text/html" }).end("<html><body><h1>page</h1></body></html>"));
|
|
@@ -84,7 +84,11 @@ async function capture(pathname, generations) {
|
|
|
84
84
|
}
|
|
85
85
|
|
|
86
86
|
function getCanvasImage(html) {
|
|
87
|
-
|
|
87
|
+
// either property name is correct: core pins every background longhand on the canvas, and once
|
|
88
|
+
// background-attachment joined them CSSOM had a complete set and serialised them as the
|
|
89
|
+
// `background` shorthand. Only cssText changed — core reads the value back through
|
|
90
|
+
// getPropertyValue("background-image"), which is why the round trip above still holds
|
|
91
|
+
const match = html.match(/background(?:-image)?:\s*url\(["']?data:image\/png;base64,([^"')]+)["']?\)/);
|
|
88
92
|
return match && match[1];
|
|
89
93
|
}
|
|
90
94
|
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/* global URL */
|
|
2
|
+
|
|
3
|
+
// --dump-json exists because --output-json replaces the page with its JSON: given an explicit
|
|
4
|
+
// output path it wrote <output>.json and no page file at all, so a script that wanted the saved
|
|
5
|
+
// page AND its metadata got neither. --dump-json writes the metadata to stdout and leaves the page
|
|
6
|
+
// file exactly where it would have been, so the two outputs combine.
|
|
7
|
+
//
|
|
8
|
+
// Two things have to hold for that to be worth anything. The page file must still be written where
|
|
9
|
+
// it was asked for, and the JSON must NOT carry the page content, which is what made the
|
|
10
|
+
// --output-json file 34 MB in the first place. The network info is the third: it is captured only
|
|
11
|
+
// when the JSON is asked for, so a flag that reaches the writer but not the capture would emit a
|
|
12
|
+
// JSON with the request and response fields silently missing.
|
|
13
|
+
|
|
14
|
+
import { test } from "node:test";
|
|
15
|
+
import assert from "node:assert/strict";
|
|
16
|
+
import { createServer } from "node:http";
|
|
17
|
+
import { execFile } from "node:child_process";
|
|
18
|
+
import { promisify } from "node:util";
|
|
19
|
+
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
20
|
+
import { tmpdir } from "node:os";
|
|
21
|
+
import { join } from "node:path";
|
|
22
|
+
import process from "node:process";
|
|
23
|
+
import { cliDirectory } from "../target.js";
|
|
24
|
+
|
|
25
|
+
const execFileAsync = promisify(execFile);
|
|
26
|
+
const TEST_TIMEOUT = 120000;
|
|
27
|
+
const PAGE = "<html><head><title>Dumped Page</title></head><body><a href=\"/other\">other</a></body></html>";
|
|
28
|
+
|
|
29
|
+
let capturePromise;
|
|
30
|
+
|
|
31
|
+
test("--dump-json still writes the page file where it was asked for", { timeout: TEST_TIMEOUT }, async () => {
|
|
32
|
+
const { pageContent } = await getCaptureResult();
|
|
33
|
+
assert.match(pageContent, /<title>Dumped Page<\/title>/, "the page file is missing or is not the saved page");
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
test("--dump-json writes the metadata to stdout as JSON", { timeout: TEST_TIMEOUT }, async () => {
|
|
37
|
+
const { metadata } = await getCaptureResult();
|
|
38
|
+
assert.equal(metadata.title, "Dumped Page");
|
|
39
|
+
assert.ok(Array.isArray(metadata.links), "the links are missing from the dumped metadata");
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("--dump-json leaves the page content out of the JSON", { timeout: TEST_TIMEOUT }, async () => {
|
|
43
|
+
const { metadata } = await getCaptureResult();
|
|
44
|
+
assert.equal(metadata.content, undefined, "the page content was dumped, which is what --output-json already does");
|
|
45
|
+
assert.equal(metadata.binaryContent, undefined, "the page content was dumped as base64");
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("--dump-json reaches the network capture, not only the writer", { timeout: TEST_TIMEOUT }, async () => {
|
|
49
|
+
const { metadata } = await getCaptureResult();
|
|
50
|
+
assert.ok(metadata.response, "no response info: the flag reached the writer but not the capture");
|
|
51
|
+
assert.equal(metadata.response.status, 200);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("--dump-json is refused with --output-json", { timeout: TEST_TIMEOUT }, async () => {
|
|
55
|
+
const { stderr } = await runCapture(["--dump-json", "--output-json"], { expectFailure: true });
|
|
56
|
+
assert.match(stderr, /--dump-json is not compatible with --output-json/);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test("--dump-json is refused with --dump-content writing to the same stream", { timeout: TEST_TIMEOUT }, async () => {
|
|
60
|
+
const { stderr } = await runCapture(["--dump-json", "--dump-content"], { expectFailure: true, noOutput: true });
|
|
61
|
+
assert.match(stderr, /--dump-json is not compatible with --dump-content/);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
function getCaptureResult() {
|
|
65
|
+
if (!capturePromise) {
|
|
66
|
+
capturePromise = runDumpCapture();
|
|
67
|
+
}
|
|
68
|
+
return capturePromise;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function runDumpCapture() {
|
|
72
|
+
const { stdout, outputPath, directory } = await runCapture(["--dump-json"], { keep: true });
|
|
73
|
+
try {
|
|
74
|
+
return { metadata: JSON.parse(stdout), pageContent: (await readFile(outputPath)).toString("utf8") };
|
|
75
|
+
} finally {
|
|
76
|
+
await rm(directory, { recursive: true });
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async function runCapture(extraArguments, { expectFailure = false, noOutput = false, keep = false } = {}) {
|
|
81
|
+
const server = createServer((request, response) => {
|
|
82
|
+
const { pathname } = new URL(request.url, "http://localhost");
|
|
83
|
+
if (pathname === "/") {
|
|
84
|
+
response.writeHead(200, { "content-type": "text/html" }).end(PAGE);
|
|
85
|
+
} else {
|
|
86
|
+
response.writeHead(404).end();
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
await new Promise(resolve => server.listen(0, "localhost", resolve));
|
|
90
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
91
|
+
const outputPath = join(directory, "page.html");
|
|
92
|
+
try {
|
|
93
|
+
const args = ["single-file-node.js", "http://localhost:" + server.address().port + "/"];
|
|
94
|
+
if (!noOutput) {
|
|
95
|
+
args.push(outputPath);
|
|
96
|
+
}
|
|
97
|
+
const { stdout, stderr } = await execFileAsync(process.execPath, args.concat(extraArguments), { cwd: cliDirectory, maxBuffer: 64 * 1024 * 1024 });
|
|
98
|
+
assert.ok(!expectFailure, "the run was expected to fail, stderr: " + stderr);
|
|
99
|
+
return { stdout, stderr, outputPath, directory };
|
|
100
|
+
} catch (error) {
|
|
101
|
+
if (!expectFailure) {
|
|
102
|
+
throw error;
|
|
103
|
+
}
|
|
104
|
+
return { stdout: error.stdout || "", stderr: error.stderr || "", outputPath, directory };
|
|
105
|
+
} finally {
|
|
106
|
+
server.close();
|
|
107
|
+
if (!keep) {
|
|
108
|
+
await rm(directory, { recursive: true });
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/* global URL */
|
|
2
|
+
|
|
3
|
+
// CSS orders @font-face sources two ways at once, and a saved page has to honour both.
|
|
4
|
+
// Within one rule the browser uses the FIRST source that loads. Across rules sharing the same
|
|
5
|
+
// family, style, weight and stretch the rules form a single COMPOSITE face: they are checked in
|
|
6
|
+
// reverse declaration order for each character, so a character the last rule's font lacks is drawn
|
|
7
|
+
// from an earlier rule's font, under that rule's own descriptors. Measured in Chrome with two
|
|
8
|
+
// generated fonts, one mapping A-Z and one mapping only A: both are fetched, the later font draws
|
|
9
|
+
// A, the earlier draws B, and each glyph carries its own rule's size-adjust. A control with the
|
|
10
|
+
// same page and only the later rule draws B in serif and never fetches the other font, so the
|
|
11
|
+
// fallback is real and not an eager download. size-adjust does NOT split the composite face.
|
|
12
|
+
// SingleFile merged every rule sharing a font key into one array built with unshift, which reversed
|
|
13
|
+
// both axes at once, and a later fix dropped the earlier rule outright, which lost every glyph only
|
|
14
|
+
// its font carried. The cases below pin the axes against each other, because a fix that satisfies
|
|
15
|
+
// only one of them is the bug in the other direction.
|
|
16
|
+
|
|
17
|
+
import { test } from "node:test";
|
|
18
|
+
import assert from "node:assert/strict";
|
|
19
|
+
import { createServer } from "node:http";
|
|
20
|
+
import { execFile } from "node:child_process";
|
|
21
|
+
import { promisify } from "node:util";
|
|
22
|
+
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
23
|
+
import { tmpdir } from "node:os";
|
|
24
|
+
import { join, dirname } from "node:path";
|
|
25
|
+
import { fileURLToPath } from "node:url";
|
|
26
|
+
import process from "node:process";
|
|
27
|
+
import { Buffer } from "node:buffer";
|
|
28
|
+
import { cliDirectory } from "../target.js";
|
|
29
|
+
|
|
30
|
+
const execFileAsync = promisify(execFile);
|
|
31
|
+
const TEST_TIMEOUT = 120000;
|
|
32
|
+
const FONTS_DIRECTORY = join(dirname(fileURLToPath(import.meta.url)), "..", "fidelity", "pages", "fonts");
|
|
33
|
+
// three distinct fixtures, told apart in the save by their byte length
|
|
34
|
+
const FONT_NAMES = ["band.ttf", "bar.ttf", "block.ttf"];
|
|
35
|
+
// the two Dup rules differ in size-adjust, which getFontKey does not cover, so they share a key
|
|
36
|
+
// while rendering differently. That is what makes per-rule pairing the thing to test: any reduction
|
|
37
|
+
// that keeps one rule pairs its metrics with the other rule's font
|
|
38
|
+
const PAGE = "<html><head><style>" +
|
|
39
|
+
"@font-face{font-family:\"Dup\";src:url(/fonts/band.ttf) format(\"truetype\");size-adjust:50%}" +
|
|
40
|
+
"@font-face{font-family:\"Dup\";src:url(/fonts/bar.ttf) format(\"truetype\");size-adjust:150%}" +
|
|
41
|
+
"@font-face{font-family:\"Solo\";src:url(/fonts/block.ttf) format(\"truetype\"),url(/fonts/band.ttf) format(\"truetype\")}" +
|
|
42
|
+
"h1{font-family:\"Dup\",serif}p{font-family:\"Solo\",serif}" +
|
|
43
|
+
"</style></head><body><h1>Head</h1><p>Body</p></body></html>";
|
|
44
|
+
|
|
45
|
+
let capturePromise;
|
|
46
|
+
|
|
47
|
+
test("every @font-face rule of a composite face is kept", { timeout: TEST_TIMEOUT }, async () => {
|
|
48
|
+
const { embedded } = await getCaptureResult();
|
|
49
|
+
const dup = embedded.filter(rule => rule.family === "Dup");
|
|
50
|
+
assert.equal(dup.length, 2, "a rule of the composite face was dropped, so it lost the glyphs only its font carries");
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("each rule of a composite face keeps its own source", { timeout: TEST_TIMEOUT }, async () => {
|
|
54
|
+
const { embedded, sizes } = await getCaptureResult();
|
|
55
|
+
const { earlier, later } = getCompositeRules(embedded);
|
|
56
|
+
assert.equal(earlier.length, sizes["band.ttf"],
|
|
57
|
+
`the size-adjust:50% rule embedded ${describe(earlier.length, sizes)} instead of band.ttf, its own source`);
|
|
58
|
+
assert.equal(later.length, sizes["bar.ttf"],
|
|
59
|
+
`the size-adjust:150% rule embedded ${describe(later.length, sizes)} instead of bar.ttf, its own source`);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test("a composite face keeps its declaration order", { timeout: TEST_TIMEOUT }, async () => {
|
|
63
|
+
const { embedded } = await getCaptureResult();
|
|
64
|
+
const { earlier, later } = getCompositeRules(embedded);
|
|
65
|
+
assert.ok(embedded.indexOf(earlier) < embedded.indexOf(later),
|
|
66
|
+
"the rules were reordered, which inverts which font the browser reaches for first");
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
function getCompositeRules(embedded) {
|
|
70
|
+
const dup = embedded.filter(rule => rule.family === "Dup");
|
|
71
|
+
const earlier = dup.find(rule => /size-adjust:\s*50%/.test(rule.text));
|
|
72
|
+
const later = dup.find(rule => /size-adjust:\s*150%/.test(rule.text));
|
|
73
|
+
assert.ok(earlier, "no rule carries the earlier rule's size-adjust:50%");
|
|
74
|
+
assert.ok(later, "no rule carries the later rule's size-adjust:150%");
|
|
75
|
+
return { earlier, later };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
test("a single @font-face rule still resolves to its first source", { timeout: TEST_TIMEOUT }, async () => {
|
|
79
|
+
const { embedded, sizes } = await getCaptureResult();
|
|
80
|
+
const solo = embedded.filter(rule => rule.family === "Solo");
|
|
81
|
+
assert.equal(solo.length, 1, "expected one Solo rule");
|
|
82
|
+
assert.equal(solo[0].length, sizes["block.ttf"],
|
|
83
|
+
`the Solo rule embedded ${describe(solo[0].length, sizes)} instead of block.ttf, its first source`);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
function describe(length, sizes) {
|
|
87
|
+
const name = Object.keys(sizes).find(fontName => sizes[fontName] === length);
|
|
88
|
+
return name || length + " bytes";
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function getCaptureResult() {
|
|
92
|
+
if (!capturePromise) {
|
|
93
|
+
capturePromise = runCapture();
|
|
94
|
+
}
|
|
95
|
+
return capturePromise;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async function runCapture() {
|
|
99
|
+
const fonts = new Map();
|
|
100
|
+
const sizes = {};
|
|
101
|
+
for (const name of FONT_NAMES) {
|
|
102
|
+
const data = await readFile(join(FONTS_DIRECTORY, name));
|
|
103
|
+
fonts.set(name, data);
|
|
104
|
+
sizes[name] = data.length;
|
|
105
|
+
}
|
|
106
|
+
const server = createServer((request, response) => {
|
|
107
|
+
const { pathname } = new URL(request.url, "http://localhost");
|
|
108
|
+
const name = pathname.startsWith("/fonts/") && pathname.slice("/fonts/".length);
|
|
109
|
+
if (name && fonts.has(name)) {
|
|
110
|
+
response.writeHead(200, { "content-type": "font/ttf" }).end(fonts.get(name));
|
|
111
|
+
} else if (pathname === "/") {
|
|
112
|
+
response.writeHead(200, { "content-type": "text/html" }).end(PAGE);
|
|
113
|
+
} else {
|
|
114
|
+
response.writeHead(404).end();
|
|
115
|
+
}
|
|
116
|
+
});
|
|
117
|
+
await new Promise(resolve => server.listen(0, "localhost", resolve));
|
|
118
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
119
|
+
try {
|
|
120
|
+
const outputPath = join(directory, "out.html");
|
|
121
|
+
await execFileAsync(process.execPath, ["single-file-node.js", "http://localhost:" + server.address().port + "/", outputPath], { cwd: cliDirectory });
|
|
122
|
+
const content = (await readFile(outputPath)).toString("utf8");
|
|
123
|
+
const embedded = [...content.matchAll(/@font-face\s*{[^}]*}/g)].map(match => {
|
|
124
|
+
const family = match[0].match(/font-family:\s*"?([^;"}]+)"?/);
|
|
125
|
+
const data = match[0].match(/base64,([A-Za-z0-9+/=]+)/);
|
|
126
|
+
return {
|
|
127
|
+
family: family && family[1].trim(),
|
|
128
|
+
length: data ? Buffer.from(data[1], "base64").length : 0,
|
|
129
|
+
text: match[0].replace(/base64,[A-Za-z0-9+/=]+/, "base64,...")
|
|
130
|
+
};
|
|
131
|
+
});
|
|
132
|
+
return { embedded, sizes };
|
|
133
|
+
} finally {
|
|
134
|
+
await rm(directory, { recursive: true });
|
|
135
|
+
server.close();
|
|
136
|
+
}
|
|
137
|
+
}
|
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
// so a plain save embedded the HTML as src:url(data:text/html;base64,...) inside the @font-face,
|
|
6
6
|
// while --compress-content dropped it because only that path tests the bytes with FontFace. An
|
|
7
7
|
// HTML body at a media URL is never a usable resource, so both modes now treat it like a 404.
|
|
8
|
+
// A @font-face left with no source at all is dropped whole, rather than kept as an "src:" with
|
|
9
|
+
// nothing after it, which is the invalid declaration a browser discards anyway.
|
|
8
10
|
//
|
|
9
11
|
// The controls are a real font and a real image on the same page: they must still be embedded.
|
|
10
12
|
|
|
@@ -39,7 +41,7 @@ const capturePromises = new Map();
|
|
|
39
41
|
test("an HTML page served at a font URL is dropped from a plain save", { timeout: TEST_TIMEOUT }, async () => {
|
|
40
42
|
const content = (await getCaptureResult(false)).toString("utf8");
|
|
41
43
|
assert.ok(!content.includes("data:text/html"), "an HTML body was embedded as a resource");
|
|
42
|
-
assert.
|
|
44
|
+
assert.doesNotMatch(content, /@font-face\s*{[^}]*font-family:\s*"?Fake"?/, "the unusable @font-face rule was kept");
|
|
43
45
|
assert.match(content, /font-family:\s*"?Real"?;\s*src:\s*url\("?data:font\/ttf;base64,/, "the real font was not embedded");
|
|
44
46
|
});
|
|
45
47
|
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/* global URL */
|
|
2
|
+
|
|
3
|
+
// --insert-meta-noindex is a core option that had no CLI flag at all until now, so nothing here
|
|
4
|
+
// ever proved it arrives. That is the failure this file guards: an option can be declared, parsed
|
|
5
|
+
// and documented and still do nothing, because the value has to survive the whole trip from the
|
|
6
|
+
// command line into the page's own context.
|
|
7
|
+
//
|
|
8
|
+
// The baseline capture is the half that makes it mean something: without the flag the element may
|
|
9
|
+
// not appear, otherwise the assertion above would pass on a page that was going to carry it anyway.
|
|
10
|
+
//
|
|
11
|
+
// Its neighbour insertCanonicalLink deliberately has NO flag. It sits in the same core list, but
|
|
12
|
+
// single-file.js forces it to true on every capture, so a flag for it would be a switch that reads
|
|
13
|
+
// as configurable and cannot turn anything off.
|
|
14
|
+
|
|
15
|
+
import { test } from "node:test";
|
|
16
|
+
import assert from "node:assert/strict";
|
|
17
|
+
import { createServer } from "node:http";
|
|
18
|
+
import { execFile } from "node:child_process";
|
|
19
|
+
import { promisify } from "node:util";
|
|
20
|
+
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
21
|
+
import { tmpdir } from "node:os";
|
|
22
|
+
import { join } from "node:path";
|
|
23
|
+
import process from "node:process";
|
|
24
|
+
import { cliDirectory } from "../target.js";
|
|
25
|
+
|
|
26
|
+
const execFileAsync = promisify(execFile);
|
|
27
|
+
const TEST_TIMEOUT = 120000;
|
|
28
|
+
const PAGE = "<html><head><title>Head Page</title></head><body>content</body></html>";
|
|
29
|
+
const ROBOTS_META = /<meta[^>]*name=["']?robots["']?[^>]*content=["']?noindex/i;
|
|
30
|
+
|
|
31
|
+
const capturePromises = new Map();
|
|
32
|
+
|
|
33
|
+
test("--insert-meta-noindex inserts the robots meta", { timeout: TEST_TIMEOUT }, async () => {
|
|
34
|
+
const content = await getCaptureResult("inserted");
|
|
35
|
+
assert.match(content, ROBOTS_META, "the robots meta is missing, so the flag never reached core");
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test("the robots meta is not inserted without the flag", { timeout: TEST_TIMEOUT }, async () => {
|
|
39
|
+
const content = await getCaptureResult("baseline");
|
|
40
|
+
assert.doesNotMatch(content, ROBOTS_META, "the robots meta appears without --insert-meta-noindex");
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
function getCaptureResult(variant) {
|
|
44
|
+
if (!capturePromises.has(variant)) {
|
|
45
|
+
capturePromises.set(variant, runCapture(variant == "inserted" ? ["--insert-meta-noindex"] : []));
|
|
46
|
+
}
|
|
47
|
+
return capturePromises.get(variant);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async function runCapture(extraArguments) {
|
|
51
|
+
const server = createServer((request, response) => {
|
|
52
|
+
const { pathname } = new URL(request.url, "http://localhost");
|
|
53
|
+
if (pathname === "/") {
|
|
54
|
+
response.writeHead(200, { "content-type": "text/html" }).end(PAGE);
|
|
55
|
+
} else {
|
|
56
|
+
response.writeHead(404).end();
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
await new Promise(resolve => server.listen(0, "localhost", resolve));
|
|
60
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
61
|
+
try {
|
|
62
|
+
const outputPath = join(directory, "page.html");
|
|
63
|
+
await execFileAsync(process.execPath, [
|
|
64
|
+
"single-file-node.js", "http://localhost:" + server.address().port + "/", outputPath
|
|
65
|
+
].concat(extraArguments), { cwd: cliDirectory });
|
|
66
|
+
return (await readFile(outputPath)).toString("utf8");
|
|
67
|
+
} finally {
|
|
68
|
+
await rm(directory, { recursive: true });
|
|
69
|
+
server.close();
|
|
70
|
+
}
|
|
71
|
+
}
|