single-file-cli 2.7.0 → 2.7.2

package/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.7.0",
+  "version": "2.7.2",
   "description": "SingleFile CLI",
   "exports": {
     ".": "./single-file-cli-api.js"

package/lib/cdp-client.js

@@ -605,7 +605,7 @@ async function waitForPageReadyState({ Page }, state, { options, debugMessages }
 			Page.removeEventListener(FRAME_NAVIGATED_EVENT_TYPE, onFrameNavigated);
 		};
 		const onLifecycleEvent = createLifecycleEventHandler(state, timeoutState, resolve, cleanup, { options, debugMessages });
-		const onFrameNavigated = createFrameNavigatedHandler(timeoutState, reject, cleanup, { options, debugMessages });
+		const onFrameNavigated = createFrameNavigatedHandler(state, timeoutState, reject, cleanup, { options, debugMessages });
 		state.settle = () => {
 			clearTimeout(timeoutState.timeoutId);
 			cleanup();
@@ -618,17 +618,20 @@ async function waitForPageReadyState({ Page }, state, { options, debugMessages }
 
 function createLifecycleEventHandler(state, timeoutState, resolve, cleanup, { options, debugMessages }) {
 	return ({ params }) => {
-		const { frameId, name } = params;
-		if (frameId === state.topFrameId) {
-			logData(["Detecting lifecycle event", name], { options, debugMessages });
-			const stateIndex = NETWORK_STATES.indexOf(name);
-			if (stateIndex != -1 && (state.reachedStateIndex == -1 || stateIndex < state.reachedStateIndex)) {
-				state.reachedStateIndex = stateIndex;
-			}
+		const { frameId, loaderId, name } = params;
+		// the frame keeps its ID across documents: the events of the blank page,
+		// whose network goes idle while the server has not answered yet, would
+		// satisfy the wait and the page would be captured as soon as it commits
+		if (frameId !== state.topFrameId || loaderId !== state.loaderId) {
+			return;
+		}
+		logData(["Detecting lifecycle event", name], { options, debugMessages });
+		const stateIndex = NETWORK_STATES.indexOf(name);
+		if (stateIndex != -1 && (state.reachedStateIndex == -1 || stateIndex < state.reachedStateIndex)) {
+			state.reachedStateIndex = stateIndex;
 		}
-		const shouldResolve = frameId === state.topFrameId &&
-			(name === options.browserWaitUntil ||
-				(timeoutState.timeoutId && NETWORK_STATES.indexOf(name) < NETWORK_STATES.indexOf(options.browserWaitUntil)));
+		const shouldResolve = name === options.browserWaitUntil ||
+			(timeoutState.timeoutId && NETWORK_STATES.indexOf(name) < NETWORK_STATES.indexOf(options.browserWaitUntil));
 		if (shouldResolve) {
 			// the delay is restarted when the page reaches a further state, so
 			// that it is captured once it stopped settling
@@ -643,15 +646,25 @@ function createLifecycleEventHandler(state, timeoutState, resolve, cleanup, { op
 	};
 };
 
-function createFrameNavigatedHandler(timeoutState, reject, cleanup, { options, debugMessages }) {
+function createFrameNavigatedHandler(state, timeoutState, reject, cleanup, { options, debugMessages }) {
 	const UNREACHABLE_URL_ERROR_MESSAGE = "Unreachable URL";
 	return ({ params }) => {
 		const { frame } = params;
-		if (!frame.parentId && frame.unreachableUrl) {
-			logData(["Detecting unreachable URL", frame.unreachableUrl], { options, debugMessages });
-			clearTimeout(timeoutState.timeoutId);
-			cleanup();
-			reject(new Error(UNREACHABLE_URL_ERROR_MESSAGE + ": " + frame.unreachableUrl));
+		if (!frame.parentId) {
+			if (frame.unreachableUrl) {
+				logData(["Detecting unreachable URL", frame.unreachableUrl], { options, debugMessages });
+				clearTimeout(timeoutState.timeoutId);
+				cleanup();
+				reject(new Error(UNREACHABLE_URL_ERROR_MESSAGE + ": " + frame.unreachableUrl));
+			} else {
+				// a new document starts the wait over: the states the previous one
+				// reached and the delay it started say nothing about this one
+				logData(["Detecting navigation", frame.url], { options, debugMessages });
+				clearTimeout(timeoutState.timeoutId);
+				timeoutState.timeoutId = undefined;
+				state.reachedStateIndex = -1;
+				state.loaderId = frame.loaderId;
+			}
 		}
 	};
 }

package/lib/version.js

@@ -1 +1 @@
-export const version = "2.7.0";
+export const version = "2.7.2";

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "single-file-cli",
-	"version": "2.7.0",
+	"version": "2.7.2",
 	"description": "SingleFile CLI",
 	"author": "Gildas Lormeau",
 	"license": "AGPL-3.0-or-later",
@@ -17,7 +17,8 @@
 	"scripts": {
 		"test": "node --test --test-concurrency=2 \"test/**/*.test.js\"",
 		"test:dev": "SINGLE_FILE_TARGET=dev node --test --test-concurrency=2 \"test/**/*.test.js\"",
-		"lint": "eslint ."
+		"lint": "eslint .",
+		"update-core": "bash update-core.sh"
 	},
 	"bin": {
 		"single-file": "single-file-node.js"

package/test/e2e/navigation.test.js

@@ -0,0 +1,43 @@
+/* global setTimeout */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, rm, readFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import process from "node:process";
+import { cliDirectory } from "../target.js";
+
+const execFileAsync = promisify(execFile);
+
+// The first page redirects itself once its network went quiet, inside the
+// settle delay of the wait. The second page keeps its body for longer than
+// that delay, so a capture decided by the first page's state saves it empty.
+test("a page that navigates after its network went quiet is captured once the new page loaded", { timeout: 120000 }, async () => {
+	const server = createServer((request, response) => {
+		response.writeHead(200, { "content-type": "text/html" });
+		if (request.url == "/second") {
+			response.write("<!doctype html><title>second</title><p>second page head</p>");
+			setTimeout(() => response.end("<p id=late>second page body arrived late</p>"), 4000);
+		} else {
+			response.end("<!doctype html><title>first</title><p id=first>first page content</p><script>setTimeout(() => location.href = '/second', 800)</script>");
+		}
+	});
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const url = "http://localhost:" + server.address().port + "/";
+		const output = join(directory, "out.html");
+		await execFileAsync(process.execPath, ["single-file-node.js", url, output], { cwd: cliDirectory });
+		const content = await readFile(output, "utf8");
+		assert.ok(content.includes("second page body arrived late"), "the page was captured before the second page loaded");
+		assert.ok(!content.includes("first page content"), "the first page was captured");
+	} finally {
+		await rm(directory, { recursive: true });
+		server.closeAllConnections();
+		server.close();
+	}
+});

package/test/e2e/slow-response.test.js

@@ -0,0 +1,45 @@
+/* global setTimeout */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, rm, readFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import process from "node:process";
+import { cliDirectory } from "../target.js";
+
+const execFileAsync = promisify(execFile);
+
+// The server answers after the network of the blank page went idle, and sends
+// the body of the page later still. A wait satisfied by the blank page captures
+// the document as soon as it commits, while it has a head and no body yet.
+test("a page whose server answers late is captured once loaded", { timeout: 120000 }, async () => {
+	const server = createServer((request, response) => {
+		if (request.url == "/") {
+			setTimeout(() => {
+				response.writeHead(200, { "content-type": "text/html" });
+				response.write("<!doctype html><html><head><title>late</title>");
+				setTimeout(() => response.end("</head><body><p id=late>body arrived late</p></body></html>"), 3000);
+			}, 3000);
+		} else {
+			response.writeHead(404);
+			response.end();
+		}
+	});
+	await new Promise(resolve => server.listen(0, "localhost", resolve));
+	const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
+	try {
+		const url = "http://localhost:" + server.address().port + "/";
+		const output = join(directory, "out.html");
+		await execFileAsync(process.execPath, ["single-file-node.js", url, output], { cwd: cliDirectory });
+		const content = await readFile(output, "utf8");
+		assert.ok(content.includes("body arrived late"), "the page was captured before its body arrived");
+	} finally {
+		await rm(directory, { recursive: true });
+		server.closeAllConnections();
+		server.close();
+	}
+});

package/update-core.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -e
+
+version="${1:-$(npm view single-file-core version)}"
+sed -i.bak "s#^CORE_PACKAGE=\"npm:single-file-core@.*\"#CORE_PACKAGE=\"npm:single-file-core@$version\"#" build.sh
+rm build.sh.bak
+bash build.sh
+echo "single-file-core $version"