single-file-cli 2.10.0 → 2.11.0

package/lib/version.js

@@ -1 +1 @@
-export const version = "2.10.0";
+export const version = "2.11.0";

package/options.js

@@ -141,7 +141,7 @@ const OPTIONS_INFO = [{
 	"extract-data-from-page": { description: "Extract compressed data from the page instead of fetching the page in order to create universal self-extracting HTML files", type: "boolean", defaultValue: true },
 	"prevent-appended-data": { description: "Prevent appending data after the compressed data when creating self-extracting HTML files", type: "boolean" },
 	"declare-appended-data": { description: "Declare the data appended after the compressed data as the comment of the ZIP archive, for readers rejecting undeclared trailing bytes (e.g. java.util.zip); ZIP tools then print that data when listing the archive. Has no effect when the extra data is relocated ahead of the compressed data, because nothing is appended then", type: "boolean" },
-	"max-appended-data-length": { description: "Maximum number of bytes appended after the compressed data when creating self-extracting HTML files, 16361 by default. Readers only tolerate trailing bytes as far back as their end-of-archive scan reaches (16383 bytes for libarchive, 32768 for perl Archive::Zip, 65557 for Python zipfile); the default fits the narrowest. Data over the limit is relocated ahead of the compressed data instead", type: "number" },
+	"max-appended-data-length": { description: "Maximum number of bytes appended after the compressed data when creating self-extracting HTML files, 16361 by default. Readers only tolerate trailing bytes as far back as their end-of-archive scan reaches (16383 bytes for libarchive, 32768 for perl Archive::Zip, 65557 for Python zipfile); the default fits the narrowest. Data over the limit is relocated ahead of the compressed data instead. Raising the limit above 65535 silently defeats --declare-appended-data: the declaration is the length of the ZIP comment, a 16-bit field, so a longer run is appended undeclared and without a warning", type: "number" },
 	"embed-screenshot": { description: "Embed a screenshot of the page as a PNG file in the compressed file (self-extracting HTML or ZIP file). When enabled, the resulting file can be read as a ZIP file or a PNG image.", type: "boolean" },
 	"embed-screenshot-options": { description: "Options passed to the CDP method `Page.captureScreenshot()` given as a JSON string (e.g. { \"captureBeyondViewport\": false })", type: "string" },
 	"embedded-image": { description: "Path to a PNG image to embed in the compressed file. Unlike --embed-screenshot it is also compatible with --crawl-save-archive, where it becomes the image of the whole archive.", type: "string" },
@@ -193,6 +193,7 @@ const OPTIONS_INFO = [{
 }, {
 	"include-bom": { key: "includeBOM", description: "Include the UTF-8 BOM into the HTML page, ignored when the page is compressed unless --extract-data-from-page is disabled and no image is embedded", type: "boolean" },
 	"insert-meta-csp": { key: "insertMetaCSP", description: "Include a <meta> tag with a CSP to avoid potential requests to internet when viewing a page", type: "boolean", defaultValue: true },
+	"insert-meta-noindex": { key: "insertMetaNoIndex", description: "Insert a <meta name=robots content=noindex> element, so a saved page served on a public host is not indexed. Ignored when the page already declares noindex", type: "boolean" },
 	"remove-saved-date": { description: "Remove saved date metadata in HTML header", type: "boolean" },
 	"save-original-urls": { key: "saveOriginalURLs", description: "Save the original URLS in the embedded contents", type: "boolean" },
 	"insert-single-file-comment": { description: "Insert a comment in the HTML header with the URL of the page", type: "boolean", defaultValue: true },
@@ -212,6 +213,7 @@ const OPTIONS_INFO = [{
 
 	"save-raw-page": { description: "Save the original page without interpreting it into the browser", type: "boolean" },
 	"output-json": { description: "Output the result as a JSON string containing the page and network info", type: "boolean" },
+	"dump-json": { description: "Write the same JSON as --output-json to stdout, without the page content, and leave the page file alone. Unlike --output-json the page is still written where it would have been, so the two outputs can be combined. Incompatible with --output-json, and with --dump-content unless --output is set, because both write to stdout", type: "boolean" },
 	"create-browser-profile": { description: "Path of the browser profile directory to create or update instead of saving a page. The browser is started with a visible window on the URL passed as argument, log in to the website and quit the browser to save the profile, then pass it to --browser-profile when saving pages", type: "string" },
 
 }, {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "2.10.0",
+	"version": "2.11.0",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"license": "AGPL-3.0-or-later",

package/single-file-cli-api.js

@@ -77,6 +77,12 @@ async function initialize(options) {
 	if ((options.embedPdf || options.embeddedPdf || options.embedScreenshot || options.embeddedImage) && !options.compressContent) {
 		throw new Error("--embed-pdf, --embedded-pdf, --embed-screenshot and --embedded-image require --compress-content");
 	}
+	if (options.dumpJson && options.outputJson) {
+		throw new Error("--dump-json is not compatible with --output-json, which already writes the JSON in place of the page");
+	}
+	if (options.dumpJson && options.dumpContent && !options.output) {
+		throw new Error("--dump-json is not compatible with --dump-content unless --output is set, because both write to stdout");
+	}
 	if (options.crawlSaveArchiveDedup && !options.crawlSaveArchive) {
 		throw new Error("--crawl-save-archive-dedup requires --crawl-save-archive");
 	}
@@ -400,6 +406,7 @@ async function capturePage(options) {
 		if (options.archiveFilename) {
 			await writeFile(options.archiveFilename, content);
 			pageData.archiveFilename = options.archiveFilename;
+			dumpJsonMetadata(pageData, options);
 			return pageData;
 		}
 		if (options.outputJson) {
@@ -429,6 +436,7 @@ async function capturePage(options) {
 			const outputDirectory = getOutputDirectory(options);
 			pageData.filename = filename.startsWith(outputDirectory) ? filename.substring(outputDirectory.length) : filename;
 		}
+		dumpJsonMetadata(pageData, options);
 		return pageData;
 	} catch (error) {
 		errorCount++;
@@ -453,6 +461,18 @@ async function capturePage(options) {
 	}
 }
 
+function dumpJsonMetadata(pageData, options) {
+	if (options.dumpJson) {
+		const metadata = Object.assign({}, pageData);
+		delete metadata.content;
+		delete metadata.binaryContent;
+		delete metadata.doctype;
+		delete metadata.viewport;
+		delete metadata.comment;
+		console.log(JSON.stringify(metadata, null, 2)); // eslint-disable-line no-console
+	}
+}
+
 function getOutputDirectory(options) {
 	if (Array.isArray(options.outputDirectory)) {
 		const outputDirectory = options.outputDirectory.pop();

package/test/e2e/dump-json.test.js

@@ -0,0 +1,111 @@
+/* global URL */
+
+// --dump-json exists because --output-json replaces the page with its JSON: given an explicit
+// output path it wrote <output>.json and no page file at all, so a script that wanted the saved
+// page AND its metadata got neither. --dump-json writes the metadata to stdout and leaves the page
+// file exactly where it would have been, so the two outputs combine.
+//
+// Two things have to hold for that to be worth anything. The page file must still be written where
+// it was asked for, and the JSON must NOT carry the page content, which is what made the
+// --output-json file 34 MB in the first place. The network info is the third: it is captured only
+// when the JSON is asked for, so a flag that reaches the writer but not the capture would emit a
+// JSON with the request and response fields silently missing.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import process from "node:process";
+import { cliDirectory } from "../target.js";
+
+const execFileAsync = promisify(execFile);
+const TEST_TIMEOUT = 120000;
+const PAGE = "<html><head><title>Dumped Page</title></head><body><a href=\"/other\">other</a></body></html>";
+
+let capturePromise;
+
+test("--dump-json still writes the page file where it was asked for", { timeout: TEST_TIMEOUT }, async () => {
+	const { pageContent } = await getCaptureResult();
+	assert.match(pageContent, /<title>Dumped Page<\/title>/, "the page file is missing or is not the saved page");
+});
+
+test("--dump-json writes the metadata to stdout as JSON", { timeout: TEST_TIMEOUT }, async () => {
+	const { metadata } = await getCaptureResult();
+	assert.equal(metadata.title, "Dumped Page");
+	assert.ok(Array.isArray(metadata.links), "the links are missing from the dumped metadata");
+});
+
+test("--dump-json leaves the page content out of the JSON", { timeout: TEST_TIMEOUT }, async () => {
+	const { metadata } = await getCaptureResult();
+	assert.equal(metadata.content, undefined, "the page content was dumped, which is what --output-json already does");
+	assert.equal(metadata.binaryContent, undefined, "the page content was dumped as base64");
+});
+
+test("--dump-json reaches the network capture, not only the writer", { timeout: TEST_TIMEOUT }, async () => {
+	const { metadata } = await getCaptureResult();
+	assert.ok(metadata.response, "no response info: the flag reached the writer but not the capture");
+	assert.equal(metadata.response.status, 200);
+});
+
+test("--dump-json is refused with --output-json", { timeout: TEST_TIMEOUT }, async () => {
+	const { stderr } = await runCapture(["--dump-json", "--output-json"], { expectFailure: true });
+	assert.match(stderr, /--dump-json is not compatible with --output-json/);
+});
+
+test("--dump-json is refused with --dump-content writing to the same stream", { timeout: TEST_TIMEOUT }, async () => {
+	const { stderr } = await runCapture(["--dump-json", "--dump-content"], { expectFailure: true, noOutput: true });
+	assert.match(stderr, /--dump-json is not compatible with --dump-content/);
+});
+
+function getCaptureResult() {
+	if (!capturePromise) {
+		capturePromise = runDumpCapture();
+	}
+	return capturePromise;
+}
+
+async function runDumpCapture() {
+	const { stdout, outputPath, directory } = await runCapture(["--dump-json"], { keep: true });
+	try {
+		return { metadata: JSON.parse(stdout), pageContent: (await readFile(outputPath)).toString("utf8") };
+	} finally {
+		await rm(directory, { recursive: true });
+	}
+}
+
+async function runCapture(extraArguments, { expectFailure = false, noOutput = false, keep = false } = {}) {
+	const server = createServer((request, response) => {
+		const { pathname } = new URL(request.url, "http://localhost");
+		if (pathname === "/") {
+			response.writeHead(200, { "content-type": "text/html" }).end(PAGE);
+		} else {
+			response.writeHead(404).end();
+		}
+	});
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	const outputPath = join(directory, "page.html");
+	try {
+		const args = ["single-file-node.js", "http://localhost:" + server.address().port + "/"];
+		if (!noOutput) {
+			args.push(outputPath);
+		}
+		const { stdout, stderr } = await execFileAsync(process.execPath, args.concat(extraArguments), { cwd: cliDirectory, maxBuffer: 64 * 1024 * 1024 });
+		assert.ok(!expectFailure, "the run was expected to fail, stderr: " + stderr);
+		return { stdout, stderr, outputPath, directory };
+	} catch (error) {
+		if (!expectFailure) {
+			throw error;
+		}
+		return { stdout: error.stdout || "", stderr: error.stderr || "", outputPath, directory };
+	} finally {
+		server.close();
+		if (!keep) {
+			await rm(directory, { recursive: true });
+		}
+	}
+}

package/test/e2e/font-face-source-order.test.js

@@ -1,14 +1,18 @@
 /* global URL */
 
 // CSS orders @font-face sources two ways at once, and a saved page has to honour both.
-// Within one rule the browser uses the FIRST source that loads. Across duplicate rules sharing the
-// same descriptors the LAST rule wins outright: measured in Chrome, only the later rule's font is
-// fetched and document.fonts reports the earlier face as "unloaded", so it is never a fallback.
+// Within one rule the browser uses the FIRST source that loads. Across rules sharing the same
+// family, style, weight and stretch the rules form a single COMPOSITE face: they are checked in
+// reverse declaration order for each character, so a character the last rule's font lacks is drawn
+// from an earlier rule's font, under that rule's own descriptors. Measured in Chrome with two
+// generated fonts, one mapping A-Z and one mapping only A: both are fetched, the later font draws
+// A, the earlier draws B, and each glyph carries its own rule's size-adjust. A control with the
+// same page and only the later rule draws B in serif and never fetches the other font, so the
+// fallback is real and not an eager download. size-adjust does NOT split the composite face.
 // SingleFile merged every rule sharing a font key into one array built with unshift, which reversed
-// both axes at once. The within-rule axis then read correctly and the across-rule axis backwards,
-// so the save embedded the font from the rule the browser had overridden — the right font was
-// absent from the file entirely. The two cases below pin the axes against each other, because a fix
-// that satisfies only one of them is the bug in the other direction.
+// both axes at once, and a later fix dropped the earlier rule outright, which lost every glyph only
+// its font carried. The cases below pin the axes against each other, because a fix that satisfies
+// only one of them is the bug in the other direction.
 
 import { test } from "node:test";
 import assert from "node:assert/strict";
@@ -29,8 +33,8 @@ const FONTS_DIRECTORY = join(dirname(fileURLToPath(import.meta.url)), "..", "fid
 // three distinct fixtures, told apart in the save by their byte length
 const FONT_NAMES = ["band.ttf", "bar.ttf", "block.ttf"];
 // the two Dup rules differ in size-adjust, which getFontKey does not cover, so they share a key
-// while rendering differently. That is what makes "keep the last rule" the only correct reduction:
-// keeping the first would pair the later rule's font with the earlier rule's metrics
+// while rendering differently. That is what makes per-rule pairing the thing to test: any reduction
+// that keeps one rule pairs its metrics with the other rule's font
 const PAGE = "<html><head><style>" +
 	"@font-face{font-family:\"Dup\";src:url(/fonts/band.ttf) format(\"truetype\");size-adjust:50%}" +
 	"@font-face{font-family:\"Dup\";src:url(/fonts/bar.ttf) format(\"truetype\");size-adjust:150%}" +
@@ -40,22 +44,37 @@ const PAGE = "<html><head><style>" +
 
 let capturePromise;
 
-test("duplicate @font-face rules resolve to the later rule, as the browser does", { timeout: TEST_TIMEOUT }, async () => {
-	const { embedded, sizes } = await getCaptureResult();
+test("every @font-face rule of a composite face is kept", { timeout: TEST_TIMEOUT }, async () => {
+	const { embedded } = await getCaptureResult();
 	const dup = embedded.filter(rule => rule.family === "Dup");
-	assert.ok(dup.length > 0, "no Dup rule survived the save");
-	dup.forEach(rule => assert.equal(rule.length, sizes["bar.ttf"],
-		`a Dup rule embedded ${describe(rule.length, sizes)} instead of bar.ttf, the later rule's font`));
+	assert.equal(dup.length, 2, "a rule of the composite face was dropped, so it lost the glyphs only its font carries");
+});
+
+test("each rule of a composite face keeps its own source", { timeout: TEST_TIMEOUT }, async () => {
+	const { embedded, sizes } = await getCaptureResult();
+	const { earlier, later } = getCompositeRules(embedded);
+	assert.equal(earlier.length, sizes["band.ttf"],
+		`the size-adjust:50% rule embedded ${describe(earlier.length, sizes)} instead of band.ttf, its own source`);
+	assert.equal(later.length, sizes["bar.ttf"],
+		`the size-adjust:150% rule embedded ${describe(later.length, sizes)} instead of bar.ttf, its own source`);
 });
 
-test("a shadowed @font-face rule is dropped, with its own descriptors", { timeout: TEST_TIMEOUT }, async () => {
+test("a composite face keeps its declaration order", { timeout: TEST_TIMEOUT }, async () => {
 	const { embedded } = await getCaptureResult();
-	const dup = embedded.filter(rule => rule.family === "Dup");
-	assert.equal(dup.length, 1, "the shadowed rule was kept, so the font is embedded twice");
-	assert.match(dup[0].text, /size-adjust:\s*150%/, "the surviving rule is not the later one");
-	assert.doesNotMatch(dup[0].text, /size-adjust:\s*50%/, "the earlier rule's metrics survived");
+	const { earlier, later } = getCompositeRules(embedded);
+	assert.ok(embedded.indexOf(earlier) < embedded.indexOf(later),
+		"the rules were reordered, which inverts which font the browser reaches for first");
 });
 
+function getCompositeRules(embedded) {
+	const dup = embedded.filter(rule => rule.family === "Dup");
+	const earlier = dup.find(rule => /size-adjust:\s*50%/.test(rule.text));
+	const later = dup.find(rule => /size-adjust:\s*150%/.test(rule.text));
+	assert.ok(earlier, "no rule carries the earlier rule's size-adjust:50%");
+	assert.ok(later, "no rule carries the later rule's size-adjust:150%");
+	return { earlier, later };
+}
+
 test("a single @font-face rule still resolves to its first source", { timeout: TEST_TIMEOUT }, async () => {
 	const { embedded, sizes } = await getCaptureResult();
 	const solo = embedded.filter(rule => rule.family === "Solo");

package/test/e2e/insert-head-elements.test.js

@@ -0,0 +1,71 @@
+/* global URL */
+
+// --insert-meta-noindex is a core option that had no CLI flag at all until now, so nothing here
+// ever proved it arrives. That is the failure this file guards: an option can be declared, parsed
+// and documented and still do nothing, because the value has to survive the whole trip from the
+// command line into the page's own context.
+//
+// The baseline capture is the half that makes it mean something: without the flag the element may
+// not appear, otherwise the assertion above would pass on a page that was going to carry it anyway.
+//
+// Its neighbour insertCanonicalLink deliberately has NO flag. It sits in the same core list, but
+// single-file.js forces it to true on every capture, so a flag for it would be a switch that reads
+// as configurable and cannot turn anything off.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import process from "node:process";
+import { cliDirectory } from "../target.js";
+
+const execFileAsync = promisify(execFile);
+const TEST_TIMEOUT = 120000;
+const PAGE = "<html><head><title>Head Page</title></head><body>content</body></html>";
+const ROBOTS_META = /<meta[^>]*name=["']?robots["']?[^>]*content=["']?noindex/i;
+
+const capturePromises = new Map();
+
+test("--insert-meta-noindex inserts the robots meta", { timeout: TEST_TIMEOUT }, async () => {
+	const content = await getCaptureResult("inserted");
+	assert.match(content, ROBOTS_META, "the robots meta is missing, so the flag never reached core");
+});
+
+test("the robots meta is not inserted without the flag", { timeout: TEST_TIMEOUT }, async () => {
+	const content = await getCaptureResult("baseline");
+	assert.doesNotMatch(content, ROBOTS_META, "the robots meta appears without --insert-meta-noindex");
+});
+
+function getCaptureResult(variant) {
+	if (!capturePromises.has(variant)) {
+		capturePromises.set(variant, runCapture(variant == "inserted" ? ["--insert-meta-noindex"] : []));
+	}
+	return capturePromises.get(variant);
+}
+
+async function runCapture(extraArguments) {
+	const server = createServer((request, response) => {
+		const { pathname } = new URL(request.url, "http://localhost");
+		if (pathname === "/") {
+			response.writeHead(200, { "content-type": "text/html" }).end(PAGE);
+		} else {
+			response.writeHead(404).end();
+		}
+	});
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const outputPath = join(directory, "page.html");
+		await execFileAsync(process.execPath, [
+			"single-file-node.js", "http://localhost:" + server.address().port + "/", outputPath
+		].concat(extraArguments), { cwd: cliDirectory });
+		return (await readFile(outputPath)).toString("utf8");
+	} finally {
+		await rm(directory, { recursive: true });
+		server.close();
+	}
+}