single-file-cli 2.0.83 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +30 -0
- package/build.sh +1 -1
- package/compile.sh +18 -13
- package/deno.json +2 -3
- package/deno.lock +545 -56
- package/lib/browser.js +58 -15
- package/lib/cdp-client-util.js +6 -6
- package/lib/cdp-client.js +151 -94
- package/lib/deno-polyfill.js +25 -30
- package/lib/single-file-bundle.js +1 -1
- package/lib/single-file-script.js +3 -1
- package/lib/version.js +1 -1
- package/options.js +2 -1
- package/package.json +13 -8
- package/single-file-cli-api.js +48 -26
- package/single-file-launcher.js +11 -2
- package/test/e2e/crawl.test.js +144 -0
- package/test/e2e/frame-gate.test.js +62 -0
- package/test/e2e/output-json.test.js +45 -0
- package/test/unit/command.test.js +41 -0
- package/test/unit/file-url.test.js +37 -0
|
@@ -68,7 +68,9 @@ function initSingleFile(constants) {
|
|
|
68
68
|
const pendingRequest = pendingRequests.get(requestId);
|
|
69
69
|
if (pendingRequest) {
|
|
70
70
|
pendingRequests.delete(requestId);
|
|
71
|
-
|
|
71
|
+
const fetchError = new Error(error.error);
|
|
72
|
+
fetchError.code = error.code;
|
|
73
|
+
pendingRequest.reject(fetchError);
|
|
72
74
|
}
|
|
73
75
|
};
|
|
74
76
|
|
package/lib/version.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const version = "2.
|
|
1
|
+
export const version = "2.1.1";
|
package/options.js
CHANGED
|
@@ -59,6 +59,7 @@ const OPTIONS_INFO = [{
|
|
|
59
59
|
"browser-debug": { description: "Enable debug mode", type: "boolean" },
|
|
60
60
|
"browser-arg": { description: "Argument passed to the browser", type: "string[]", alias: "browser-argument" },
|
|
61
61
|
"browser-args": { description: "Arguments provided as a JSON array and passed to the browser", type: "string" },
|
|
62
|
+
"browser-single-process": { description: "Run the browser as a single process", type: "boolean", defaultValue: true },
|
|
62
63
|
"browser-start-minimized": { description: "Minimize the browser", type: "boolean" },
|
|
63
64
|
"browser-ignore-insecure-certs": { description: "Ignore HTTPs errors", type: "boolean" },
|
|
64
65
|
"browser-remote-debugging-URL": { description: "Remote debugging URL", type: "string" }
|
|
@@ -138,7 +139,7 @@ const OPTIONS_INFO = [{
|
|
|
138
139
|
"embed-pdf-options": { description: "Options passed to the CDP method `Page.printToPDF()` given as a JSON string (e.g. { \"pageRanges\": \"1-1\", \"paperWidth\": 11, \"paperHeight\": 8.5 })", type: "string" },
|
|
139
140
|
"embedded-pdf": { description: "Path to a PDF file to embed in the compressed file.", type: "string" }
|
|
140
141
|
}, {
|
|
141
|
-
"filename-template": { description: "Template used to generate the output filename (see
|
|
142
|
+
"filename-template": { description: "Template used to generate the output filename (see https://github.com/gildas-lormeau/SingleFile/wiki/Template-variables-and-functions)", type: "string", defaultValue: "%if-empty<{page-title}|No title> ({date-locale} {time-locale}).{filename-extension}" },
|
|
142
143
|
"filename-conflict-action": { description: "Action when the filename is conflicting with existing one on the filesystem. The possible values are \"uniquify\" (default), \"overwrite\" and \"skip\"", type: "string", defaultValue: "uniquify" },
|
|
143
144
|
"filename-replacement-character": { description: "The character used for replacing invalid characters in filenames", type: "string", defaultValue: "_" },
|
|
144
145
|
"filename-replaced-character": { description: "The replaced character and the character(s) used for replacing the replacement character in filenames separated by a space, e.g. --filename-replaced-character \">\" _GT_ to replace \">\" with _GT_", type: "string[]", defaultValue: ["~ ~", "+ +", "? ?", "% %", "* *", ": :", "| |", "\" "", "< <", "> >", "\\\\ \", "\\x00-\\x1f _", "\x7F _"] },
|
package/package.json
CHANGED
|
@@ -1,22 +1,27 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "single-file-cli",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.1.1",
|
|
4
4
|
"description": "SingleFile CLI",
|
|
5
5
|
"author": "Gildas Lormeau",
|
|
6
6
|
"engines": {
|
|
7
|
-
"deno": ">=
|
|
8
|
-
"bun": ">=1.
|
|
9
|
-
"node": ">=
|
|
7
|
+
"deno": ">=2.2",
|
|
8
|
+
"bun": ">=1.2",
|
|
9
|
+
"node": ">=24.0.0"
|
|
10
10
|
},
|
|
11
11
|
"type": "module",
|
|
12
|
+
"scripts": {
|
|
13
|
+
"test": "node --test \"test/**/*.test.js\"",
|
|
14
|
+
"lint": "eslint ."
|
|
15
|
+
},
|
|
12
16
|
"bin": {
|
|
13
17
|
"single-file": "single-file-node.js"
|
|
14
18
|
},
|
|
15
19
|
"dependencies": {
|
|
16
|
-
"simple-cdp": "^1.
|
|
17
|
-
"ws": "^8.18.3"
|
|
20
|
+
"simple-cdp": "^1.10.0"
|
|
18
21
|
},
|
|
19
22
|
"devDependencies": {
|
|
20
|
-
"@eslint/js": "^9.39.1"
|
|
23
|
+
"@eslint/js": "^9.39.1",
|
|
24
|
+
"esbuild": "^0.27.4",
|
|
25
|
+
"eslint": "^10.8.1"
|
|
21
26
|
}
|
|
22
|
-
}
|
|
27
|
+
}
|
package/single-file-cli-api.js
CHANGED
|
@@ -21,8 +21,9 @@
|
|
|
21
21
|
* Source.
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
|
-
/* global URL
|
|
24
|
+
/* global URL */
|
|
25
25
|
|
|
26
|
+
import { Buffer } from "node:buffer";
|
|
26
27
|
import * as backend from "./lib/cdp-client.js";
|
|
27
28
|
import { getZipScriptSource } from "./lib/single-file-script.js";
|
|
28
29
|
import { Deno, path } from "./lib/deno-polyfill.js";
|
|
@@ -65,6 +66,9 @@ export { initialize };
|
|
|
65
66
|
|
|
66
67
|
async function initialize(options) {
|
|
67
68
|
options = Object.assign({}, DEFAULT_OPTIONS, options);
|
|
69
|
+
if ((options.embedPdf || options.embeddedPdf || options.embedScreenshot || options.embeddedImage) && !options.compressContent) {
|
|
70
|
+
throw new Error("--embed-pdf, --embedded-pdf, --embed-screenshot and --embedded-image require --compress-content");
|
|
71
|
+
}
|
|
68
72
|
maxParallelWorkers = options.maxParallelWorkers || 8;
|
|
69
73
|
try {
|
|
70
74
|
await backend.initialize(options);
|
|
@@ -83,7 +87,8 @@ async function initialize(options) {
|
|
|
83
87
|
}
|
|
84
88
|
if (options.crawlSyncSession || options.crawlLoadSession) {
|
|
85
89
|
try {
|
|
86
|
-
tasks = JSON.parse(await readTextFile(options.crawlSyncSession || options.crawlLoadSession))
|
|
90
|
+
tasks = JSON.parse(await readTextFile(options.crawlSyncSession || options.crawlLoadSession))
|
|
91
|
+
.map(task => Object.assign({ originalUrls: [task.url] }, task));
|
|
87
92
|
} catch (error) {
|
|
88
93
|
if (options.crawlLoadSession) {
|
|
89
94
|
throw error;
|
|
@@ -127,17 +132,20 @@ async function finish(options) {
|
|
|
127
132
|
if (options.crawlReplaceURLs && !options.compressContent) {
|
|
128
133
|
for (const task of tasks) {
|
|
129
134
|
try {
|
|
130
|
-
|
|
135
|
+
const outputFilename = getOutputDirectory(options) + task.filename;
|
|
136
|
+
let pageContent = await readTextFile(outputFilename);
|
|
131
137
|
tasks.forEach(otherTask => {
|
|
132
138
|
if (otherTask.filename) {
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
139
|
+
otherTask.originalUrls.forEach(originalUrl => {
|
|
140
|
+
pageContent = pageContent.replace(new RegExp(escapeRegExp("\"" + originalUrl + "\""), "gi"), "\"" + otherTask.filename + "\"");
|
|
141
|
+
pageContent = pageContent.replace(new RegExp(escapeRegExp("'" + originalUrl + "'"), "gi"), "'" + otherTask.filename + "'");
|
|
142
|
+
const filename = otherTask.filename.replace(/ /g, "%20");
|
|
143
|
+
pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + originalUrl + " "), "gi"), "=" + filename + " ");
|
|
144
|
+
pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + originalUrl + ">"), "gi"), "=" + filename + ">");
|
|
145
|
+
});
|
|
138
146
|
}
|
|
139
147
|
});
|
|
140
|
-
await writeTextFile(
|
|
148
|
+
await writeTextFile(outputFilename, pageContent);
|
|
141
149
|
} catch {
|
|
142
150
|
// ignored
|
|
143
151
|
}
|
|
@@ -174,12 +182,11 @@ async function runNextTask() {
|
|
|
174
182
|
if (options.crawlLinks && testMaxDepth(task)) {
|
|
175
183
|
const urls = pageData.links;
|
|
176
184
|
let newTasks = await Promise.all(urls.map(url => createTask(url, options, task, task.rootTaskURL || task.url)));
|
|
177
|
-
newTasks = newTasks.filter(task => task &&
|
|
185
|
+
newTasks = newTasks.filter((task, taskIndex) => task &&
|
|
178
186
|
testMaxDepth(task) &&
|
|
179
|
-
!tasks.find(otherTask => otherTask.url == task.url) &&
|
|
180
|
-
!newTasks.find(otherTask => otherTask != task && otherTask.url == task.url) &&
|
|
181
187
|
(!options.crawlInnerLinksOnly || task.isInnerLink) &&
|
|
182
|
-
(!options.crawlNoParent || (task.isChild || !task.isInnerLink))
|
|
188
|
+
(!options.crawlNoParent || (task.isChild || !task.isInnerLink)) &&
|
|
189
|
+
!mergeDuplicateTask(task, tasks.concat(newTasks.slice(0, taskIndex))));
|
|
183
190
|
tasks.splice(tasks.length, 0, ...newTasks);
|
|
184
191
|
}
|
|
185
192
|
}
|
|
@@ -188,6 +195,18 @@ async function runNextTask() {
|
|
|
188
195
|
}
|
|
189
196
|
}
|
|
190
197
|
|
|
198
|
+
function mergeDuplicateTask(task, otherTasks) {
|
|
199
|
+
const duplicateTask = otherTasks.find(otherTask => otherTask && otherTask.url == task.url);
|
|
200
|
+
if (duplicateTask) {
|
|
201
|
+
task.originalUrls.forEach(url => {
|
|
202
|
+
if (!duplicateTask.originalUrls.includes(url)) {
|
|
203
|
+
duplicateTask.originalUrls.push(url);
|
|
204
|
+
}
|
|
205
|
+
});
|
|
206
|
+
return true;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
191
210
|
function testMaxDepth(task) {
|
|
192
211
|
const options = task.options;
|
|
193
212
|
return (options.crawlMaxDepth == 0 || task.depth <= options.crawlMaxDepth) &&
|
|
@@ -195,10 +214,13 @@ function testMaxDepth(task) {
|
|
|
195
214
|
}
|
|
196
215
|
|
|
197
216
|
async function createTask(url, options, parentTask, rootTaskURL) {
|
|
198
|
-
|
|
217
|
+
const originalUrl = url;
|
|
199
218
|
url = parentTask ? rewriteURL(url, options.crawlRemoveURLFragment, options.crawlRewriteRules) : url;
|
|
200
219
|
if (url) {
|
|
201
220
|
if (!VALID_URL_TEST.test(url)) {
|
|
221
|
+
if (parentTask) {
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
202
224
|
try {
|
|
203
225
|
url = url.replace(/\\/g, "/");
|
|
204
226
|
url = url.replace(/#/g, "%23");
|
|
@@ -215,7 +237,7 @@ async function createTask(url, options, parentTask, rootTaskURL) {
|
|
|
215
237
|
url,
|
|
216
238
|
isInnerLink,
|
|
217
239
|
isChild,
|
|
218
|
-
|
|
240
|
+
originalUrls: [originalUrl],
|
|
219
241
|
rootTaskURL,
|
|
220
242
|
depth: parentTask ? parentTask.depth + 1 : 0,
|
|
221
243
|
externalLinkDepth: isInnerLink ? -1 : parentTask ? parentTask.externalLinkDepth + 1 : -1,
|
|
@@ -252,7 +274,7 @@ function rewriteURL(url, crawlRemoveURLFragment, crawlRewriteRules = []) {
|
|
|
252
274
|
|
|
253
275
|
function getHostURL(url) {
|
|
254
276
|
url = new URL(url);
|
|
255
|
-
return url.protocol + "//" + (url.username ? url.username + (url.password
|
|
277
|
+
return url.protocol + "//" + (url.username ? url.username + (url.password ? ":" + url.password : "") + "@" : "") + url.host + "/";
|
|
256
278
|
}
|
|
257
279
|
|
|
258
280
|
async function capturePage(options) {
|
|
@@ -270,14 +292,8 @@ async function capturePage(options) {
|
|
|
270
292
|
}
|
|
271
293
|
if (options.outputJson) {
|
|
272
294
|
if (content instanceof Uint8Array) {
|
|
273
|
-
const fileReader = new FileReader();
|
|
274
|
-
fileReader.readAsDataURL(new Blob([content]));
|
|
275
|
-
content = await new Promise(resolve => {
|
|
276
|
-
fileReader.onload = () => resolve(fileReader.result);
|
|
277
|
-
});
|
|
278
|
-
content = content.replace(/^data:.*?;base64,/, "");
|
|
279
295
|
pageData.content = undefined;
|
|
280
|
-
pageData.binaryContent = content;
|
|
296
|
+
pageData.binaryContent = Buffer.from(content).toString("base64");
|
|
281
297
|
}
|
|
282
298
|
pageData.doctype = undefined;
|
|
283
299
|
pageData.viewport = undefined;
|
|
@@ -308,6 +324,8 @@ async function capturePage(options) {
|
|
|
308
324
|
} else {
|
|
309
325
|
await writeTextFile(filename, content);
|
|
310
326
|
}
|
|
327
|
+
const outputDirectory = getOutputDirectory(options);
|
|
328
|
+
pageData.filename = filename.startsWith(outputDirectory) ? filename.substring(outputDirectory.length) : filename;
|
|
311
329
|
}
|
|
312
330
|
return pageData;
|
|
313
331
|
} catch (error) {
|
|
@@ -332,7 +350,7 @@ async function capturePage(options) {
|
|
|
332
350
|
}
|
|
333
351
|
}
|
|
334
352
|
|
|
335
|
-
|
|
353
|
+
function getOutputDirectory(options) {
|
|
336
354
|
if (Array.isArray(options.outputDirectory)) {
|
|
337
355
|
const outputDirectory = options.outputDirectory.pop();
|
|
338
356
|
if (outputDirectory.startsWith("/")) {
|
|
@@ -345,9 +363,13 @@ async function getFilename(filename, options, index = 1) {
|
|
|
345
363
|
if (outputDirectory && !outputDirectory.endsWith("/")) {
|
|
346
364
|
outputDirectory += "/";
|
|
347
365
|
}
|
|
348
|
-
|
|
366
|
+
return outputDirectory;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
async function getFilename(filename, options, index = 1) {
|
|
370
|
+
let newFilename = getOutputDirectory(options) + filename;
|
|
349
371
|
if (options.filenameConflictAction == "overwrite") {
|
|
350
|
-
return
|
|
372
|
+
return newFilename;
|
|
351
373
|
} else if (options.filenameConflictAction == "uniquify" && index > 1) {
|
|
352
374
|
const regExpMatchExtension = /(\.[^.]+)$/;
|
|
353
375
|
const matchExtension = newFilename.match(regExpMatchExtension);
|
package/single-file-launcher.js
CHANGED
|
@@ -30,12 +30,12 @@ const { readTextFile, readFile, exit, addSignalListener } = Deno;
|
|
|
30
30
|
|
|
31
31
|
try {
|
|
32
32
|
addSignalListener("SIGTERM", closeBrowserAndExit);
|
|
33
|
-
} catch
|
|
33
|
+
} catch {
|
|
34
34
|
// ignored
|
|
35
35
|
}
|
|
36
36
|
try {
|
|
37
37
|
addSignalListener("SIGINT", closeBrowserAndExit);
|
|
38
|
-
} catch
|
|
38
|
+
} catch {
|
|
39
39
|
// ignored
|
|
40
40
|
}
|
|
41
41
|
|
|
@@ -84,6 +84,15 @@ async function run() {
|
|
|
84
84
|
options.browserCookies = parseCookies(cookiesContent);
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
|
+
if (options.emulateMediaFeatures) {
|
|
88
|
+
options.emulateMediaFeatures = options.emulateMediaFeatures.map(feature => {
|
|
89
|
+
const colonIndex = feature.indexOf(":");
|
|
90
|
+
return {
|
|
91
|
+
name: feature.substring(0, colonIndex),
|
|
92
|
+
value: feature.substring(colonIndex + 1)
|
|
93
|
+
};
|
|
94
|
+
});
|
|
95
|
+
}
|
|
87
96
|
if (options.httpHeaders) {
|
|
88
97
|
const headers = {};
|
|
89
98
|
for (const header of options.httpHeaders) {
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/* global URL */
|
|
2
|
+
|
|
3
|
+
import { test } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import { createServer } from "node:http";
|
|
6
|
+
import { execFile } from "node:child_process";
|
|
7
|
+
import { promisify } from "node:util";
|
|
8
|
+
import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
|
|
9
|
+
import { tmpdir } from "node:os";
|
|
10
|
+
import { join, dirname } from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
import process from "node:process";
|
|
13
|
+
|
|
14
|
+
const execFileAsync = promisify(execFile);
|
|
15
|
+
const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
16
|
+
const TEST_TIMEOUT = 120000;
|
|
17
|
+
|
|
18
|
+
let crawlPromise;
|
|
19
|
+
|
|
20
|
+
test("a page linked with and without fragment is captured once", { timeout: TEST_TIMEOUT }, async () => {
|
|
21
|
+
const { filenames, stderr } = await getCrawlResult();
|
|
22
|
+
assert.ok(filenames.includes("Top Page.html"), "missing top page, captured: " + filenames.join(", ") + ", stderr: " + stderr);
|
|
23
|
+
assert.equal(filenames.filter(filename => filename.startsWith("Linked Page")).length, 1);
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
test("links to pages with the same title are rewritten to distinct filenames", { timeout: TEST_TIMEOUT }, async () => {
|
|
27
|
+
const { filenames, pages } = await getCrawlResult();
|
|
28
|
+
assert.ok(filenames.includes("Same Title.html"));
|
|
29
|
+
assert.ok(filenames.includes("Same Title (2).html"));
|
|
30
|
+
assert.ok(pages["Top Page.html"].includes("=Same%20Title.html>"));
|
|
31
|
+
assert.ok(pages["Top Page.html"].includes("=Same%20Title%20(2).html>"));
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("links to another port are not crawled as inner links", { timeout: TEST_TIMEOUT }, async () => {
|
|
35
|
+
const { filenames, otherServerRequests } = await getCrawlResult();
|
|
36
|
+
assert.ok(!filenames.includes("Other Server.html"));
|
|
37
|
+
assert.equal(otherServerRequests.length, 0);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("fragment links are rewritten to the captured filename", { timeout: TEST_TIMEOUT }, async () => {
|
|
41
|
+
const { pages } = await getCrawlResult();
|
|
42
|
+
assert.ok(pages["Top Page.html"].includes("=Fragment%20Page.html>"));
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test("all link variants of a deduplicated page are rewritten", { timeout: TEST_TIMEOUT }, async () => {
|
|
46
|
+
const { pages } = await getCrawlResult();
|
|
47
|
+
assert.ok(!pages["Top Page.html"].includes("page.html#section"));
|
|
48
|
+
assert.equal((pages["Top Page.html"].match(/=Linked%20Page\.html>/g) || []).length, 2);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test("mail and script links are not crawled as external links", { timeout: TEST_TIMEOUT }, async () => {
|
|
52
|
+
const server = createServer((_, response) => servePage(response, "Mail Top", `
|
|
53
|
+
<a href="mailto:contact@example.net">mail</a>
|
|
54
|
+
<a href="javascript:void(0)">script</a>
|
|
55
|
+
<a href="tel:+15550100">phone</a>`));
|
|
56
|
+
await new Promise(resolve => server.listen(0, "localhost", resolve));
|
|
57
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
58
|
+
try {
|
|
59
|
+
const url = "http://localhost:" + server.address().port + "/";
|
|
60
|
+
await execFileAsync(process.execPath, [
|
|
61
|
+
"single-file-node.js", url,
|
|
62
|
+
"--crawl-links",
|
|
63
|
+
"--crawl-inner-links-only=false",
|
|
64
|
+
"--output-directory", directory,
|
|
65
|
+
"--filename-template", "{page-title}.html",
|
|
66
|
+
"--errors-file", join(directory, "errors.txt"),
|
|
67
|
+
"--max-parallel-workers", "1"
|
|
68
|
+
], { cwd: cliDirectory });
|
|
69
|
+
const filenames = await readdir(directory);
|
|
70
|
+
const errors = filenames.includes("errors.txt") ? await readFile(join(directory, "errors.txt"), "utf8") : "";
|
|
71
|
+
assert.deepEqual(filenames, ["Mail Top.html"], "errors: " + errors);
|
|
72
|
+
} finally {
|
|
73
|
+
await rm(directory, { recursive: true });
|
|
74
|
+
server.close();
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
function getCrawlResult() {
|
|
79
|
+
if (!crawlPromise) {
|
|
80
|
+
crawlPromise = runCrawl();
|
|
81
|
+
}
|
|
82
|
+
return crawlPromise;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function runCrawl() {
|
|
86
|
+
const otherServerRequests = [];
|
|
87
|
+
const otherServer = createServer((request, response) => {
|
|
88
|
+
otherServerRequests.push(request.url);
|
|
89
|
+
servePage(response, "Other Server");
|
|
90
|
+
});
|
|
91
|
+
await new Promise(resolve => otherServer.listen(0, "localhost", resolve));
|
|
92
|
+
const otherServerOrigin = "http://localhost:" + otherServer.address().port;
|
|
93
|
+
const server = createServer((request, response) => {
|
|
94
|
+
const { pathname } = new URL(request.url, "http://localhost");
|
|
95
|
+
if (pathname === "/") {
|
|
96
|
+
servePage(response, "Top Page", `
|
|
97
|
+
<a href="/page.html">page</a>
|
|
98
|
+
<a href="/page.html#section">section</a>
|
|
99
|
+
<a href="/fragment.html#top">fragment</a>
|
|
100
|
+
<a href="/same-title-1.html">first</a>
|
|
101
|
+
<a href="/same-title-2.html">second</a>
|
|
102
|
+
<a href="${otherServerOrigin}/other.html">other</a>
|
|
103
|
+
<a href="mailto:contact@example.net">mail</a>`);
|
|
104
|
+
} else if (pathname === "/page.html") {
|
|
105
|
+
servePage(response, "Linked Page");
|
|
106
|
+
} else if (pathname === "/fragment.html") {
|
|
107
|
+
servePage(response, "Fragment Page");
|
|
108
|
+
} else if (pathname === "/same-title-1.html") {
|
|
109
|
+
servePage(response, "Same Title", "first");
|
|
110
|
+
} else if (pathname === "/same-title-2.html") {
|
|
111
|
+
servePage(response, "Same Title", "second");
|
|
112
|
+
} else {
|
|
113
|
+
response.writeHead(404).end();
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
await new Promise(resolve => server.listen(0, "localhost", resolve));
|
|
117
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
118
|
+
try {
|
|
119
|
+
const url = "http://localhost:" + server.address().port + "/";
|
|
120
|
+
const { stderr } = await execFileAsync(process.execPath, [
|
|
121
|
+
"single-file-node.js", url,
|
|
122
|
+
"--crawl-links",
|
|
123
|
+
"--crawl-replace-URLs",
|
|
124
|
+
"--output-directory", directory,
|
|
125
|
+
"--filename-template", "{page-title}.html",
|
|
126
|
+
"--max-parallel-workers", "1"
|
|
127
|
+
], { cwd: cliDirectory });
|
|
128
|
+
const filenames = await readdir(directory);
|
|
129
|
+
const pages = {};
|
|
130
|
+
for (const filename of filenames) {
|
|
131
|
+
pages[filename] = await readFile(join(directory, filename), "utf8");
|
|
132
|
+
}
|
|
133
|
+
return { filenames, pages, otherServerRequests, stderr };
|
|
134
|
+
} finally {
|
|
135
|
+
await rm(directory, { recursive: true });
|
|
136
|
+
server.close();
|
|
137
|
+
otherServer.close();
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function servePage(response, title, body = "") {
|
|
142
|
+
response.writeHead(200, { "content-type": "text/html" })
|
|
143
|
+
.end(`<html><head><title>${title}</title></head><body>${body}</body></html>`);
|
|
144
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/* global setTimeout, URL */
|
|
2
|
+
|
|
3
|
+
import { test } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import { createServer } from "node:http";
|
|
6
|
+
import { execFile } from "node:child_process";
|
|
7
|
+
import { promisify } from "node:util";
|
|
8
|
+
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
9
|
+
import { tmpdir } from "node:os";
|
|
10
|
+
import { join, dirname } from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
import { Buffer } from "node:buffer";
|
|
13
|
+
import process from "node:process";
|
|
14
|
+
|
|
15
|
+
const execFileAsync = promisify(execFile);
|
|
16
|
+
const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
17
|
+
const LOAD_MARKER = "MARKER_ADDED_AFTER_LOAD_EVENT";
|
|
18
|
+
const SLOW_RESOURCE_DELAY = 3500;
|
|
19
|
+
const PIXEL_PNG = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", "base64");
|
|
20
|
+
|
|
21
|
+
const TOP_PAGE = `<html><head><title>top</title></head><body>
|
|
22
|
+
<img src="/slow.png">
|
|
23
|
+
<iframe src="/frame.html"></iframe>
|
|
24
|
+
<script>
|
|
25
|
+
addEventListener("load", () => {
|
|
26
|
+
const marker = document.createElement("div");
|
|
27
|
+
marker.textContent = "${LOAD_MARKER}";
|
|
28
|
+
document.body.appendChild(marker);
|
|
29
|
+
});
|
|
30
|
+
</script>
|
|
31
|
+
</body></html>`;
|
|
32
|
+
|
|
33
|
+
test("capture waits for the top frame, not a fast iframe", { timeout: 120000 }, async () => {
|
|
34
|
+
const server = createServer((request, response) => {
|
|
35
|
+
const { pathname } = new URL(request.url, "http://localhost");
|
|
36
|
+
if (pathname === "/top.html") {
|
|
37
|
+
response.writeHead(200, { "content-type": "text/html" }).end(TOP_PAGE);
|
|
38
|
+
} else if (pathname === "/frame.html") {
|
|
39
|
+
response.writeHead(200, { "content-type": "text/html" }).end("<html><body>frame</body></html>");
|
|
40
|
+
} else if (pathname === "/slow.png") {
|
|
41
|
+
setTimeout(() => response.writeHead(200, { "content-type": "image/png" }).end(PIXEL_PNG), SLOW_RESOURCE_DELAY);
|
|
42
|
+
} else {
|
|
43
|
+
response.writeHead(404).end();
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
await new Promise(resolve => server.listen(0, "localhost", resolve));
|
|
47
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
48
|
+
try {
|
|
49
|
+
const outputPath = join(directory, "out.html");
|
|
50
|
+
const url = "http://localhost:" + server.address().port + "/top.html";
|
|
51
|
+
await execFileAsync(process.execPath, [
|
|
52
|
+
"single-file-node.js", url, outputPath,
|
|
53
|
+
"--browser-wait-until", "networkIdle",
|
|
54
|
+
"--browser-wait-until-delay", "1000"
|
|
55
|
+
], { cwd: cliDirectory });
|
|
56
|
+
const content = await readFile(outputPath, "utf8");
|
|
57
|
+
assert.ok(content.includes(LOAD_MARKER), "page was captured before the top frame finished loading");
|
|
58
|
+
} finally {
|
|
59
|
+
await rm(directory, { recursive: true });
|
|
60
|
+
server.close();
|
|
61
|
+
}
|
|
62
|
+
});
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import { execFile } from "node:child_process";
|
|
5
|
+
import { promisify } from "node:util";
|
|
6
|
+
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
7
|
+
import { tmpdir } from "node:os";
|
|
8
|
+
import { join, dirname } from "node:path";
|
|
9
|
+
import { fileURLToPath } from "node:url";
|
|
10
|
+
import { Buffer } from "node:buffer";
|
|
11
|
+
import process from "node:process";
|
|
12
|
+
|
|
13
|
+
const execFileAsync = promisify(execFile);
|
|
14
|
+
const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
15
|
+
const ZIP_SIGNATURE = "PK\u0003\u0004";
|
|
16
|
+
|
|
17
|
+
test("output-json embeds compressed content as base64", { timeout: 120000 }, async () => {
|
|
18
|
+
const server = createServer((_, response) => response
|
|
19
|
+
.writeHead(200, { "content-type": "text/html" })
|
|
20
|
+
.end("<html><head><title>JSON Output</title></head><body>content</body></html>"));
|
|
21
|
+
await new Promise(resolve => server.listen(0, "localhost", resolve));
|
|
22
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
23
|
+
try {
|
|
24
|
+
const outputPath = join(directory, "page.json");
|
|
25
|
+
const url = "http://localhost:" + server.address().port + "/";
|
|
26
|
+
const { stderr } = await execFileAsync(process.execPath, [
|
|
27
|
+
"single-file-node.js", url, outputPath,
|
|
28
|
+
"--compress-content",
|
|
29
|
+
"--output-json"
|
|
30
|
+
], { cwd: cliDirectory });
|
|
31
|
+
let rawPageData;
|
|
32
|
+
try {
|
|
33
|
+
rawPageData = await readFile(outputPath, "utf8");
|
|
34
|
+
} catch (error) {
|
|
35
|
+
throw new Error("missing output file, stderr: " + stderr, { cause: error });
|
|
36
|
+
}
|
|
37
|
+
const pageData = JSON.parse(rawPageData);
|
|
38
|
+
assert.ok(pageData.binaryContent);
|
|
39
|
+
const content = Buffer.from(pageData.binaryContent, "base64").toString("latin1");
|
|
40
|
+
assert.ok(content.includes(ZIP_SIGNATURE));
|
|
41
|
+
} finally {
|
|
42
|
+
await rm(directory, { recursive: true });
|
|
43
|
+
server.close();
|
|
44
|
+
}
|
|
45
|
+
});
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/* global setTimeout, clearTimeout */
|
|
2
|
+
|
|
3
|
+
import { test } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import process from "node:process";
|
|
6
|
+
import { Deno } from "../../lib/deno-polyfill.js";
|
|
7
|
+
|
|
8
|
+
const { Command } = Deno;
|
|
9
|
+
|
|
10
|
+
test("status resolves when the process exits with a nonzero code", async () => {
|
|
11
|
+
const command = new Command(process.execPath, { args: ["-e", "process.exit(7)"] });
|
|
12
|
+
const child = await command.spawn();
|
|
13
|
+
const status = await child.status;
|
|
14
|
+
assert.equal(status.code, 7);
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
test("status resolves when the process is killed", async () => {
|
|
18
|
+
const command = new Command(process.execPath, { args: ["-e", "setTimeout(() => {}, 60000)"] });
|
|
19
|
+
const child = await command.spawn();
|
|
20
|
+
child.kill();
|
|
21
|
+
const status = await child.status;
|
|
22
|
+
assert.equal(status.code, null);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test("a process writing large output with discarded stdio exits", async () => {
|
|
26
|
+
const script = "const chunk = \"x\".repeat(65536); for (let i = 0; i < 64; i++) { process.stdout.write(chunk); process.stderr.write(chunk); }";
|
|
27
|
+
const command = new Command(process.execPath, { args: ["-e", script], stdout: "null", stderr: "null" });
|
|
28
|
+
const child = await command.spawn();
|
|
29
|
+
let timeoutId;
|
|
30
|
+
const result = await Promise.race([
|
|
31
|
+
child.status,
|
|
32
|
+
new Promise(resolve => timeoutId = setTimeout(() => resolve("timeout"), 5000))
|
|
33
|
+
]);
|
|
34
|
+
clearTimeout(timeoutId);
|
|
35
|
+
if (result === "timeout") {
|
|
36
|
+
child.kill();
|
|
37
|
+
await child.status.catch(() => { });
|
|
38
|
+
assert.fail("process blocked on unread output");
|
|
39
|
+
}
|
|
40
|
+
assert.equal(result.code, 0);
|
|
41
|
+
});
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/* global TextDecoder */
|
|
2
|
+
|
|
3
|
+
import { test } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import { mkdtemp, writeFile, rm } from "node:fs/promises";
|
|
6
|
+
import { tmpdir } from "node:os";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { path } from "../../lib/deno-polyfill.js";
|
|
9
|
+
import { fetch as fetchWithFileSupport } from "../../lib/cdp-client-util.js";
|
|
10
|
+
|
|
11
|
+
test("fromFileUrl decodes percent-encoded characters", async () => {
|
|
12
|
+
assert.equal(await path.fromFileUrl("file:///tmp/some%20dir/page.html"), "/tmp/some dir/page.html");
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
test("fromFileUrl ignores query and fragment", async () => {
|
|
16
|
+
assert.equal(await path.fromFileUrl("file:///tmp/page.html?query=1#fragment"), "/tmp/page.html");
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
test("fetch reads a file URL with a query", async () => {
|
|
20
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
21
|
+
try {
|
|
22
|
+
const filePath = join(directory, "resource dir");
|
|
23
|
+
await writeFile(filePath, "file content");
|
|
24
|
+
const fileUrl = await path.toFileUrl(filePath);
|
|
25
|
+
const response = await fetchWithFileSupport(fileUrl + "?query=1");
|
|
26
|
+
assert.equal(response.status, 200);
|
|
27
|
+
const content = new TextDecoder().decode(await response.arrayBuffer());
|
|
28
|
+
assert.equal(content, "file content");
|
|
29
|
+
} finally {
|
|
30
|
+
await rm(directory, { recursive: true });
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("fetch returns 404 for a missing file URL", async () => {
|
|
35
|
+
const response = await fetchWithFileSupport("file:///nonexistent/path/resource");
|
|
36
|
+
assert.equal(response.status, 404);
|
|
37
|
+
});
|