single-file-cli 2.0.83 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +30 -0
- package/build.sh +1 -1
- package/compile.sh +19 -13
- package/deno.json +2 -3
- package/deno.lock +545 -56
- package/lib/browser.js +10 -11
- package/lib/cdp-client-util.js +5 -6
- package/lib/cdp-client.js +109 -90
- package/lib/deno-polyfill.js +25 -30
- package/lib/single-file-bundle.js +1 -1
- package/lib/single-file-script.js +3 -1
- package/lib/version.js +1 -1
- package/options.js +2 -1
- package/package.json +13 -8
- package/single-file-cli-api.js +47 -26
- package/single-file-launcher.js +11 -2
- package/test/e2e/crawl.test.js +142 -0
- package/test/e2e/frame-gate.test.js +61 -0
- package/test/e2e/output-json.test.js +40 -0
- package/test/unit/command.test.js +41 -0
- package/test/unit/file-url.test.js +37 -0
|
@@ -68,7 +68,9 @@ function initSingleFile(constants) {
|
|
|
68
68
|
const pendingRequest = pendingRequests.get(requestId);
|
|
69
69
|
if (pendingRequest) {
|
|
70
70
|
pendingRequests.delete(requestId);
|
|
71
|
-
|
|
71
|
+
const fetchError = new Error(error.error);
|
|
72
|
+
fetchError.code = error.code;
|
|
73
|
+
pendingRequest.reject(fetchError);
|
|
72
74
|
}
|
|
73
75
|
};
|
|
74
76
|
|
package/lib/version.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const version = "2.0
|
|
1
|
+
export const version = "2.1.0";
|
package/options.js
CHANGED
|
@@ -59,6 +59,7 @@ const OPTIONS_INFO = [{
|
|
|
59
59
|
"browser-debug": { description: "Enable debug mode", type: "boolean" },
|
|
60
60
|
"browser-arg": { description: "Argument passed to the browser", type: "string[]", alias: "browser-argument" },
|
|
61
61
|
"browser-args": { description: "Arguments provided as a JSON array and passed to the browser", type: "string" },
|
|
62
|
+
"browser-single-process": { description: "Run the browser as a single process", type: "boolean", defaultValue: true },
|
|
62
63
|
"browser-start-minimized": { description: "Minimize the browser", type: "boolean" },
|
|
63
64
|
"browser-ignore-insecure-certs": { description: "Ignore HTTPs errors", type: "boolean" },
|
|
64
65
|
"browser-remote-debugging-URL": { description: "Remote debugging URL", type: "string" }
|
|
@@ -138,7 +139,7 @@ const OPTIONS_INFO = [{
|
|
|
138
139
|
"embed-pdf-options": { description: "Options passed to the CDP method `Page.printToPDF()` given as a JSON string (e.g. { \"pageRanges\": \"1-1\", \"paperWidth\": 11, \"paperHeight\": 8.5 })", type: "string" },
|
|
139
140
|
"embedded-pdf": { description: "Path to a PDF file to embed in the compressed file.", type: "string" }
|
|
140
141
|
}, {
|
|
141
|
-
"filename-template": { description: "Template used to generate the output filename (see
|
|
142
|
+
"filename-template": { description: "Template used to generate the output filename (see https://github.com/gildas-lormeau/SingleFile/wiki/Template-variables-and-functions)", type: "string", defaultValue: "%if-empty<{page-title}|No title> ({date-locale} {time-locale}).{filename-extension}" },
|
|
142
143
|
"filename-conflict-action": { description: "Action when the filename is conflicting with existing one on the filesystem. The possible values are \"uniquify\" (default), \"overwrite\" and \"skip\"", type: "string", defaultValue: "uniquify" },
|
|
143
144
|
"filename-replacement-character": { description: "The character used for replacing invalid characters in filenames", type: "string", defaultValue: "_" },
|
|
144
145
|
"filename-replaced-character": { description: "The replaced character and the character(s) used for replacing the replacement character in filenames separated by a space, e.g. --filename-replaced-character \">\" _GT_ to replace \">\" with _GT_", type: "string[]", defaultValue: ["~ ~", "+ +", "? ?", "% %", "* *", ": :", "| |", "\" "", "< <", "> >", "\\\\ \", "\\x00-\\x1f _", "\x7F _"] },
|
package/package.json
CHANGED
|
@@ -1,22 +1,27 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "single-file-cli",
|
|
3
|
-
"version": "2.0
|
|
3
|
+
"version": "2.1.0",
|
|
4
4
|
"description": "SingleFile CLI",
|
|
5
5
|
"author": "Gildas Lormeau",
|
|
6
6
|
"engines": {
|
|
7
|
-
"deno": ">=
|
|
8
|
-
"bun": ">=1.
|
|
9
|
-
"node": ">=
|
|
7
|
+
"deno": ">=2.2",
|
|
8
|
+
"bun": ">=1.2",
|
|
9
|
+
"node": ">=24.0.0"
|
|
10
10
|
},
|
|
11
11
|
"type": "module",
|
|
12
|
+
"scripts": {
|
|
13
|
+
"test": "node --test \"test/**/*.test.js\"",
|
|
14
|
+
"lint": "eslint ."
|
|
15
|
+
},
|
|
12
16
|
"bin": {
|
|
13
17
|
"single-file": "single-file-node.js"
|
|
14
18
|
},
|
|
15
19
|
"dependencies": {
|
|
16
|
-
"simple-cdp": "^1.
|
|
17
|
-
"ws": "^8.18.3"
|
|
20
|
+
"simple-cdp": "^1.10.0"
|
|
18
21
|
},
|
|
19
22
|
"devDependencies": {
|
|
20
|
-
"@eslint/js": "^9.39.1"
|
|
23
|
+
"@eslint/js": "^9.39.1",
|
|
24
|
+
"esbuild": "^0.27.4",
|
|
25
|
+
"eslint": "^10.8.1"
|
|
21
26
|
}
|
|
22
|
-
}
|
|
27
|
+
}
|
package/single-file-cli-api.js
CHANGED
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
* Source.
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
|
-
/* global URL
|
|
24
|
+
/* global URL */
|
|
25
25
|
|
|
26
26
|
import * as backend from "./lib/cdp-client.js";
|
|
27
27
|
import { getZipScriptSource } from "./lib/single-file-script.js";
|
|
@@ -65,6 +65,9 @@ export { initialize };
|
|
|
65
65
|
|
|
66
66
|
async function initialize(options) {
|
|
67
67
|
options = Object.assign({}, DEFAULT_OPTIONS, options);
|
|
68
|
+
if ((options.embedPdf || options.embeddedPdf || options.embedScreenshot || options.embeddedImage) && !options.compressContent) {
|
|
69
|
+
throw new Error("--embed-pdf, --embedded-pdf, --embed-screenshot and --embedded-image require --compress-content");
|
|
70
|
+
}
|
|
68
71
|
maxParallelWorkers = options.maxParallelWorkers || 8;
|
|
69
72
|
try {
|
|
70
73
|
await backend.initialize(options);
|
|
@@ -83,7 +86,8 @@ async function initialize(options) {
|
|
|
83
86
|
}
|
|
84
87
|
if (options.crawlSyncSession || options.crawlLoadSession) {
|
|
85
88
|
try {
|
|
86
|
-
tasks = JSON.parse(await readTextFile(options.crawlSyncSession || options.crawlLoadSession))
|
|
89
|
+
tasks = JSON.parse(await readTextFile(options.crawlSyncSession || options.crawlLoadSession))
|
|
90
|
+
.map(task => Object.assign({ originalUrls: [task.url] }, task));
|
|
87
91
|
} catch (error) {
|
|
88
92
|
if (options.crawlLoadSession) {
|
|
89
93
|
throw error;
|
|
@@ -127,17 +131,20 @@ async function finish(options) {
|
|
|
127
131
|
if (options.crawlReplaceURLs && !options.compressContent) {
|
|
128
132
|
for (const task of tasks) {
|
|
129
133
|
try {
|
|
130
|
-
|
|
134
|
+
const outputFilename = getOutputDirectory(options) + task.filename;
|
|
135
|
+
let pageContent = await readTextFile(outputFilename);
|
|
131
136
|
tasks.forEach(otherTask => {
|
|
132
137
|
if (otherTask.filename) {
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
+
otherTask.originalUrls.forEach(originalUrl => {
|
|
139
|
+
pageContent = pageContent.replace(new RegExp(escapeRegExp("\"" + originalUrl + "\""), "gi"), "\"" + otherTask.filename + "\"");
|
|
140
|
+
pageContent = pageContent.replace(new RegExp(escapeRegExp("'" + originalUrl + "'"), "gi"), "'" + otherTask.filename + "'");
|
|
141
|
+
const filename = otherTask.filename.replace(/ /g, "%20");
|
|
142
|
+
pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + originalUrl + " "), "gi"), "=" + filename + " ");
|
|
143
|
+
pageContent = pageContent.replace(new RegExp(escapeRegExp("=" + originalUrl + ">"), "gi"), "=" + filename + ">");
|
|
144
|
+
});
|
|
138
145
|
}
|
|
139
146
|
});
|
|
140
|
-
await writeTextFile(
|
|
147
|
+
await writeTextFile(outputFilename, pageContent);
|
|
141
148
|
} catch {
|
|
142
149
|
// ignored
|
|
143
150
|
}
|
|
@@ -174,12 +181,11 @@ async function runNextTask() {
|
|
|
174
181
|
if (options.crawlLinks && testMaxDepth(task)) {
|
|
175
182
|
const urls = pageData.links;
|
|
176
183
|
let newTasks = await Promise.all(urls.map(url => createTask(url, options, task, task.rootTaskURL || task.url)));
|
|
177
|
-
newTasks = newTasks.filter(task => task &&
|
|
184
|
+
newTasks = newTasks.filter((task, taskIndex) => task &&
|
|
178
185
|
testMaxDepth(task) &&
|
|
179
|
-
!tasks.find(otherTask => otherTask.url == task.url) &&
|
|
180
|
-
!newTasks.find(otherTask => otherTask != task && otherTask.url == task.url) &&
|
|
181
186
|
(!options.crawlInnerLinksOnly || task.isInnerLink) &&
|
|
182
|
-
(!options.crawlNoParent || (task.isChild || !task.isInnerLink))
|
|
187
|
+
(!options.crawlNoParent || (task.isChild || !task.isInnerLink)) &&
|
|
188
|
+
!mergeDuplicateTask(task, tasks.concat(newTasks.slice(0, taskIndex))));
|
|
183
189
|
tasks.splice(tasks.length, 0, ...newTasks);
|
|
184
190
|
}
|
|
185
191
|
}
|
|
@@ -188,6 +194,18 @@ async function runNextTask() {
|
|
|
188
194
|
}
|
|
189
195
|
}
|
|
190
196
|
|
|
197
|
+
function mergeDuplicateTask(task, otherTasks) {
|
|
198
|
+
const duplicateTask = otherTasks.find(otherTask => otherTask && otherTask.url == task.url);
|
|
199
|
+
if (duplicateTask) {
|
|
200
|
+
task.originalUrls.forEach(url => {
|
|
201
|
+
if (!duplicateTask.originalUrls.includes(url)) {
|
|
202
|
+
duplicateTask.originalUrls.push(url);
|
|
203
|
+
}
|
|
204
|
+
});
|
|
205
|
+
return true;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
191
209
|
function testMaxDepth(task) {
|
|
192
210
|
const options = task.options;
|
|
193
211
|
return (options.crawlMaxDepth == 0 || task.depth <= options.crawlMaxDepth) &&
|
|
@@ -195,10 +213,13 @@ function testMaxDepth(task) {
|
|
|
195
213
|
}
|
|
196
214
|
|
|
197
215
|
async function createTask(url, options, parentTask, rootTaskURL) {
|
|
198
|
-
|
|
216
|
+
const originalUrl = url;
|
|
199
217
|
url = parentTask ? rewriteURL(url, options.crawlRemoveURLFragment, options.crawlRewriteRules) : url;
|
|
200
218
|
if (url) {
|
|
201
219
|
if (!VALID_URL_TEST.test(url)) {
|
|
220
|
+
if (parentTask) {
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
202
223
|
try {
|
|
203
224
|
url = url.replace(/\\/g, "/");
|
|
204
225
|
url = url.replace(/#/g, "%23");
|
|
@@ -215,7 +236,7 @@ async function createTask(url, options, parentTask, rootTaskURL) {
|
|
|
215
236
|
url,
|
|
216
237
|
isInnerLink,
|
|
217
238
|
isChild,
|
|
218
|
-
|
|
239
|
+
originalUrls: [originalUrl],
|
|
219
240
|
rootTaskURL,
|
|
220
241
|
depth: parentTask ? parentTask.depth + 1 : 0,
|
|
221
242
|
externalLinkDepth: isInnerLink ? -1 : parentTask ? parentTask.externalLinkDepth + 1 : -1,
|
|
@@ -252,7 +273,7 @@ function rewriteURL(url, crawlRemoveURLFragment, crawlRewriteRules = []) {
|
|
|
252
273
|
|
|
253
274
|
function getHostURL(url) {
|
|
254
275
|
url = new URL(url);
|
|
255
|
-
return url.protocol + "//" + (url.username ? url.username + (url.password
|
|
276
|
+
return url.protocol + "//" + (url.username ? url.username + (url.password ? ":" + url.password : "") + "@" : "") + url.host + "/";
|
|
256
277
|
}
|
|
257
278
|
|
|
258
279
|
async function capturePage(options) {
|
|
@@ -270,14 +291,8 @@ async function capturePage(options) {
|
|
|
270
291
|
}
|
|
271
292
|
if (options.outputJson) {
|
|
272
293
|
if (content instanceof Uint8Array) {
|
|
273
|
-
const fileReader = new FileReader();
|
|
274
|
-
fileReader.readAsDataURL(new Blob([content]));
|
|
275
|
-
content = await new Promise(resolve => {
|
|
276
|
-
fileReader.onload = () => resolve(fileReader.result);
|
|
277
|
-
});
|
|
278
|
-
content = content.replace(/^data:.*?;base64,/, "");
|
|
279
294
|
pageData.content = undefined;
|
|
280
|
-
pageData.binaryContent = content;
|
|
295
|
+
pageData.binaryContent = content.toBase64();
|
|
281
296
|
}
|
|
282
297
|
pageData.doctype = undefined;
|
|
283
298
|
pageData.viewport = undefined;
|
|
@@ -308,6 +323,8 @@ async function capturePage(options) {
|
|
|
308
323
|
} else {
|
|
309
324
|
await writeTextFile(filename, content);
|
|
310
325
|
}
|
|
326
|
+
const outputDirectory = getOutputDirectory(options);
|
|
327
|
+
pageData.filename = filename.startsWith(outputDirectory) ? filename.substring(outputDirectory.length) : filename;
|
|
311
328
|
}
|
|
312
329
|
return pageData;
|
|
313
330
|
} catch (error) {
|
|
@@ -332,7 +349,7 @@ async function capturePage(options) {
|
|
|
332
349
|
}
|
|
333
350
|
}
|
|
334
351
|
|
|
335
|
-
|
|
352
|
+
function getOutputDirectory(options) {
|
|
336
353
|
if (Array.isArray(options.outputDirectory)) {
|
|
337
354
|
const outputDirectory = options.outputDirectory.pop();
|
|
338
355
|
if (outputDirectory.startsWith("/")) {
|
|
@@ -345,9 +362,13 @@ async function getFilename(filename, options, index = 1) {
|
|
|
345
362
|
if (outputDirectory && !outputDirectory.endsWith("/")) {
|
|
346
363
|
outputDirectory += "/";
|
|
347
364
|
}
|
|
348
|
-
|
|
365
|
+
return outputDirectory;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
async function getFilename(filename, options, index = 1) {
|
|
369
|
+
let newFilename = getOutputDirectory(options) + filename;
|
|
349
370
|
if (options.filenameConflictAction == "overwrite") {
|
|
350
|
-
return
|
|
371
|
+
return newFilename;
|
|
351
372
|
} else if (options.filenameConflictAction == "uniquify" && index > 1) {
|
|
352
373
|
const regExpMatchExtension = /(\.[^.]+)$/;
|
|
353
374
|
const matchExtension = newFilename.match(regExpMatchExtension);
|
package/single-file-launcher.js
CHANGED
|
@@ -30,12 +30,12 @@ const { readTextFile, readFile, exit, addSignalListener } = Deno;
|
|
|
30
30
|
|
|
31
31
|
try {
|
|
32
32
|
addSignalListener("SIGTERM", closeBrowserAndExit);
|
|
33
|
-
} catch
|
|
33
|
+
} catch {
|
|
34
34
|
// ignored
|
|
35
35
|
}
|
|
36
36
|
try {
|
|
37
37
|
addSignalListener("SIGINT", closeBrowserAndExit);
|
|
38
|
-
} catch
|
|
38
|
+
} catch {
|
|
39
39
|
// ignored
|
|
40
40
|
}
|
|
41
41
|
|
|
@@ -84,6 +84,15 @@ async function run() {
|
|
|
84
84
|
options.browserCookies = parseCookies(cookiesContent);
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
|
+
if (options.emulateMediaFeatures) {
|
|
88
|
+
options.emulateMediaFeatures = options.emulateMediaFeatures.map(feature => {
|
|
89
|
+
const colonIndex = feature.indexOf(":");
|
|
90
|
+
return {
|
|
91
|
+
name: feature.substring(0, colonIndex),
|
|
92
|
+
value: feature.substring(colonIndex + 1)
|
|
93
|
+
};
|
|
94
|
+
});
|
|
95
|
+
}
|
|
87
96
|
if (options.httpHeaders) {
|
|
88
97
|
const headers = {};
|
|
89
98
|
for (const header of options.httpHeaders) {
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/* global URL */
|
|
2
|
+
|
|
3
|
+
import { test } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import { createServer } from "node:http";
|
|
6
|
+
import { execFile } from "node:child_process";
|
|
7
|
+
import { promisify } from "node:util";
|
|
8
|
+
import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
|
|
9
|
+
import { tmpdir } from "node:os";
|
|
10
|
+
import { join, dirname } from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
import process from "node:process";
|
|
13
|
+
|
|
14
|
+
const execFileAsync = promisify(execFile);
|
|
15
|
+
const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
16
|
+
const TEST_TIMEOUT = 120000;
|
|
17
|
+
|
|
18
|
+
let crawlPromise;
|
|
19
|
+
|
|
20
|
+
test("a page linked with and without fragment is captured once", { timeout: TEST_TIMEOUT }, async () => {
|
|
21
|
+
const { filenames } = await getCrawlResult();
|
|
22
|
+
assert.ok(filenames.includes("Top Page.html"));
|
|
23
|
+
assert.equal(filenames.filter(filename => filename.startsWith("Linked Page")).length, 1);
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
test("links to pages with the same title are rewritten to distinct filenames", { timeout: TEST_TIMEOUT }, async () => {
|
|
27
|
+
const { filenames, pages } = await getCrawlResult();
|
|
28
|
+
assert.ok(filenames.includes("Same Title.html"));
|
|
29
|
+
assert.ok(filenames.includes("Same Title (2).html"));
|
|
30
|
+
assert.ok(pages["Top Page.html"].includes("=Same%20Title.html>"));
|
|
31
|
+
assert.ok(pages["Top Page.html"].includes("=Same%20Title%20(2).html>"));
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("links to another port are not crawled as inner links", { timeout: TEST_TIMEOUT }, async () => {
|
|
35
|
+
const { filenames, otherServerRequests } = await getCrawlResult();
|
|
36
|
+
assert.ok(!filenames.includes("Other Server.html"));
|
|
37
|
+
assert.equal(otherServerRequests.length, 0);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("fragment links are rewritten to the captured filename", { timeout: TEST_TIMEOUT }, async () => {
|
|
41
|
+
const { pages } = await getCrawlResult();
|
|
42
|
+
assert.ok(pages["Top Page.html"].includes("=Fragment%20Page.html>"));
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test("all link variants of a deduplicated page are rewritten", { timeout: TEST_TIMEOUT }, async () => {
|
|
46
|
+
const { pages } = await getCrawlResult();
|
|
47
|
+
assert.ok(!pages["Top Page.html"].includes("page.html#section"));
|
|
48
|
+
assert.equal((pages["Top Page.html"].match(/=Linked%20Page\.html>/g) || []).length, 2);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test("mail and script links are not crawled as external links", { timeout: TEST_TIMEOUT }, async () => {
|
|
52
|
+
const server = createServer((_, response) => servePage(response, "Mail Top", `
|
|
53
|
+
<a href="mailto:contact@example.net">mail</a>
|
|
54
|
+
<a href="javascript:void(0)">script</a>
|
|
55
|
+
<a href="tel:+15550100">phone</a>`));
|
|
56
|
+
await new Promise(resolve => server.listen(0, "localhost", resolve));
|
|
57
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
58
|
+
try {
|
|
59
|
+
const url = "http://localhost:" + server.address().port + "/";
|
|
60
|
+
await execFileAsync(process.execPath, [
|
|
61
|
+
"single-file-node.js", url,
|
|
62
|
+
"--crawl-links",
|
|
63
|
+
"--crawl-inner-links-only=false",
|
|
64
|
+
"--output-directory", directory,
|
|
65
|
+
"--filename-template", "{page-title}.html",
|
|
66
|
+
"--errors-file", join(directory, "errors.txt"),
|
|
67
|
+
"--max-parallel-workers", "1"
|
|
68
|
+
], { cwd: cliDirectory });
|
|
69
|
+
assert.deepEqual(await readdir(directory), ["Mail Top.html"]);
|
|
70
|
+
} finally {
|
|
71
|
+
await rm(directory, { recursive: true });
|
|
72
|
+
server.close();
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
function getCrawlResult() {
|
|
77
|
+
if (!crawlPromise) {
|
|
78
|
+
crawlPromise = runCrawl();
|
|
79
|
+
}
|
|
80
|
+
return crawlPromise;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
async function runCrawl() {
|
|
84
|
+
const otherServerRequests = [];
|
|
85
|
+
const otherServer = createServer((request, response) => {
|
|
86
|
+
otherServerRequests.push(request.url);
|
|
87
|
+
servePage(response, "Other Server");
|
|
88
|
+
});
|
|
89
|
+
await new Promise(resolve => otherServer.listen(0, "localhost", resolve));
|
|
90
|
+
const otherServerOrigin = "http://localhost:" + otherServer.address().port;
|
|
91
|
+
const server = createServer((request, response) => {
|
|
92
|
+
const { pathname } = new URL(request.url, "http://localhost");
|
|
93
|
+
if (pathname === "/") {
|
|
94
|
+
servePage(response, "Top Page", `
|
|
95
|
+
<a href="/page.html">page</a>
|
|
96
|
+
<a href="/page.html#section">section</a>
|
|
97
|
+
<a href="/fragment.html#top">fragment</a>
|
|
98
|
+
<a href="/same-title-1.html">first</a>
|
|
99
|
+
<a href="/same-title-2.html">second</a>
|
|
100
|
+
<a href="${otherServerOrigin}/other.html">other</a>
|
|
101
|
+
<a href="mailto:contact@example.net">mail</a>`);
|
|
102
|
+
} else if (pathname === "/page.html") {
|
|
103
|
+
servePage(response, "Linked Page");
|
|
104
|
+
} else if (pathname === "/fragment.html") {
|
|
105
|
+
servePage(response, "Fragment Page");
|
|
106
|
+
} else if (pathname === "/same-title-1.html") {
|
|
107
|
+
servePage(response, "Same Title", "first");
|
|
108
|
+
} else if (pathname === "/same-title-2.html") {
|
|
109
|
+
servePage(response, "Same Title", "second");
|
|
110
|
+
} else {
|
|
111
|
+
response.writeHead(404).end();
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
await new Promise(resolve => server.listen(0, "localhost", resolve));
|
|
115
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
116
|
+
try {
|
|
117
|
+
const url = "http://localhost:" + server.address().port + "/";
|
|
118
|
+
await execFileAsync(process.execPath, [
|
|
119
|
+
"single-file-node.js", url,
|
|
120
|
+
"--crawl-links",
|
|
121
|
+
"--crawl-replace-URLs",
|
|
122
|
+
"--output-directory", directory,
|
|
123
|
+
"--filename-template", "{page-title}.html",
|
|
124
|
+
"--max-parallel-workers", "1"
|
|
125
|
+
], { cwd: cliDirectory });
|
|
126
|
+
const filenames = await readdir(directory);
|
|
127
|
+
const pages = {};
|
|
128
|
+
for (const filename of filenames) {
|
|
129
|
+
pages[filename] = await readFile(join(directory, filename), "utf8");
|
|
130
|
+
}
|
|
131
|
+
return { filenames, pages, otherServerRequests };
|
|
132
|
+
} finally {
|
|
133
|
+
await rm(directory, { recursive: true });
|
|
134
|
+
server.close();
|
|
135
|
+
otherServer.close();
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function servePage(response, title, body = "") {
|
|
140
|
+
response.writeHead(200, { "content-type": "text/html" })
|
|
141
|
+
.end(`<html><head><title>${title}</title></head><body>${body}</body></html>`);
|
|
142
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/* global setTimeout, URL */
|
|
2
|
+
|
|
3
|
+
import { test } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import { createServer } from "node:http";
|
|
6
|
+
import { execFile } from "node:child_process";
|
|
7
|
+
import { promisify } from "node:util";
|
|
8
|
+
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
9
|
+
import { tmpdir } from "node:os";
|
|
10
|
+
import { join, dirname } from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
import process from "node:process";
|
|
13
|
+
|
|
14
|
+
const execFileAsync = promisify(execFile);
|
|
15
|
+
const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
16
|
+
const LOAD_MARKER = "MARKER_ADDED_AFTER_LOAD_EVENT";
|
|
17
|
+
const SLOW_RESOURCE_DELAY = 3500;
|
|
18
|
+
const PIXEL_PNG = Uint8Array.fromBase64("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==");
|
|
19
|
+
|
|
20
|
+
const TOP_PAGE = `<html><head><title>top</title></head><body>
|
|
21
|
+
<img src="/slow.png">
|
|
22
|
+
<iframe src="/frame.html"></iframe>
|
|
23
|
+
<script>
|
|
24
|
+
addEventListener("load", () => {
|
|
25
|
+
const marker = document.createElement("div");
|
|
26
|
+
marker.textContent = "${LOAD_MARKER}";
|
|
27
|
+
document.body.appendChild(marker);
|
|
28
|
+
});
|
|
29
|
+
</script>
|
|
30
|
+
</body></html>`;
|
|
31
|
+
|
|
32
|
+
test("capture waits for the top frame, not a fast iframe", { timeout: 60000 }, async () => {
|
|
33
|
+
const server = createServer((request, response) => {
|
|
34
|
+
const { pathname } = new URL(request.url, "http://localhost");
|
|
35
|
+
if (pathname === "/top.html") {
|
|
36
|
+
response.writeHead(200, { "content-type": "text/html" }).end(TOP_PAGE);
|
|
37
|
+
} else if (pathname === "/frame.html") {
|
|
38
|
+
response.writeHead(200, { "content-type": "text/html" }).end("<html><body>frame</body></html>");
|
|
39
|
+
} else if (pathname === "/slow.png") {
|
|
40
|
+
setTimeout(() => response.writeHead(200, { "content-type": "image/png" }).end(PIXEL_PNG), SLOW_RESOURCE_DELAY);
|
|
41
|
+
} else {
|
|
42
|
+
response.writeHead(404).end();
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
await new Promise(resolve => server.listen(0, "localhost", resolve));
|
|
46
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
47
|
+
try {
|
|
48
|
+
const outputPath = join(directory, "out.html");
|
|
49
|
+
const url = "http://localhost:" + server.address().port + "/top.html";
|
|
50
|
+
await execFileAsync(process.execPath, [
|
|
51
|
+
"single-file-node.js", url, outputPath,
|
|
52
|
+
"--browser-wait-until", "networkIdle",
|
|
53
|
+
"--browser-wait-until-delay", "1000"
|
|
54
|
+
], { cwd: cliDirectory });
|
|
55
|
+
const content = await readFile(outputPath, "utf8");
|
|
56
|
+
assert.ok(content.includes(LOAD_MARKER), "page was captured before the top frame finished loading");
|
|
57
|
+
} finally {
|
|
58
|
+
await rm(directory, { recursive: true });
|
|
59
|
+
server.close();
|
|
60
|
+
}
|
|
61
|
+
});
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/* global TextDecoder */
|
|
2
|
+
|
|
3
|
+
import { test } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import { createServer } from "node:http";
|
|
6
|
+
import { execFile } from "node:child_process";
|
|
7
|
+
import { promisify } from "node:util";
|
|
8
|
+
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
9
|
+
import { tmpdir } from "node:os";
|
|
10
|
+
import { join, dirname } from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
import process from "node:process";
|
|
13
|
+
|
|
14
|
+
const execFileAsync = promisify(execFile);
|
|
15
|
+
const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
16
|
+
const ZIP_SIGNATURE = "PK\u0003\u0004";
|
|
17
|
+
|
|
18
|
+
test("output-json embeds compressed content as base64", { timeout: 60000 }, async () => {
|
|
19
|
+
const server = createServer((_, response) => response
|
|
20
|
+
.writeHead(200, { "content-type": "text/html" })
|
|
21
|
+
.end("<html><head><title>JSON Output</title></head><body>content</body></html>"));
|
|
22
|
+
await new Promise(resolve => server.listen(0, "localhost", resolve));
|
|
23
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
24
|
+
try {
|
|
25
|
+
const outputPath = join(directory, "page.json");
|
|
26
|
+
const url = "http://localhost:" + server.address().port + "/";
|
|
27
|
+
await execFileAsync(process.execPath, [
|
|
28
|
+
"single-file-node.js", url, outputPath,
|
|
29
|
+
"--compress-content",
|
|
30
|
+
"--output-json"
|
|
31
|
+
], { cwd: cliDirectory });
|
|
32
|
+
const pageData = JSON.parse(await readFile(outputPath, "utf8"));
|
|
33
|
+
assert.ok(pageData.binaryContent);
|
|
34
|
+
const content = new TextDecoder("latin1").decode(Uint8Array.fromBase64(pageData.binaryContent));
|
|
35
|
+
assert.ok(content.includes(ZIP_SIGNATURE));
|
|
36
|
+
} finally {
|
|
37
|
+
await rm(directory, { recursive: true });
|
|
38
|
+
server.close();
|
|
39
|
+
}
|
|
40
|
+
});
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/* global setTimeout, clearTimeout */
|
|
2
|
+
|
|
3
|
+
import { test } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import process from "node:process";
|
|
6
|
+
import { Deno } from "../../lib/deno-polyfill.js";
|
|
7
|
+
|
|
8
|
+
const { Command } = Deno;
|
|
9
|
+
|
|
10
|
+
test("status resolves when the process exits with a nonzero code", async () => {
|
|
11
|
+
const command = new Command(process.execPath, { args: ["-e", "process.exit(7)"] });
|
|
12
|
+
const child = await command.spawn();
|
|
13
|
+
const status = await child.status;
|
|
14
|
+
assert.equal(status.code, 7);
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
test("status resolves when the process is killed", async () => {
|
|
18
|
+
const command = new Command(process.execPath, { args: ["-e", "setTimeout(() => {}, 60000)"] });
|
|
19
|
+
const child = await command.spawn();
|
|
20
|
+
child.kill();
|
|
21
|
+
const status = await child.status;
|
|
22
|
+
assert.equal(status.code, null);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test("a process writing large output with discarded stdio exits", async () => {
|
|
26
|
+
const script = "const chunk = \"x\".repeat(65536); for (let i = 0; i < 64; i++) { process.stdout.write(chunk); process.stderr.write(chunk); }";
|
|
27
|
+
const command = new Command(process.execPath, { args: ["-e", script], stdout: "null", stderr: "null" });
|
|
28
|
+
const child = await command.spawn();
|
|
29
|
+
let timeoutId;
|
|
30
|
+
const result = await Promise.race([
|
|
31
|
+
child.status,
|
|
32
|
+
new Promise(resolve => timeoutId = setTimeout(() => resolve("timeout"), 5000))
|
|
33
|
+
]);
|
|
34
|
+
clearTimeout(timeoutId);
|
|
35
|
+
if (result === "timeout") {
|
|
36
|
+
child.kill();
|
|
37
|
+
await child.status.catch(() => { });
|
|
38
|
+
assert.fail("process blocked on unread output");
|
|
39
|
+
}
|
|
40
|
+
assert.equal(result.code, 0);
|
|
41
|
+
});
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/* global TextDecoder */
|
|
2
|
+
|
|
3
|
+
import { test } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import { mkdtemp, writeFile, rm } from "node:fs/promises";
|
|
6
|
+
import { tmpdir } from "node:os";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { path } from "../../lib/deno-polyfill.js";
|
|
9
|
+
import { fetch as fetchWithFileSupport } from "../../lib/cdp-client-util.js";
|
|
10
|
+
|
|
11
|
+
test("fromFileUrl decodes percent-encoded characters", async () => {
|
|
12
|
+
assert.equal(await path.fromFileUrl("file:///tmp/some%20dir/page.html"), "/tmp/some dir/page.html");
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
test("fromFileUrl ignores query and fragment", async () => {
|
|
16
|
+
assert.equal(await path.fromFileUrl("file:///tmp/page.html?query=1#fragment"), "/tmp/page.html");
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
test("fetch reads a file URL with a query", async () => {
|
|
20
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
21
|
+
try {
|
|
22
|
+
const filePath = join(directory, "resource dir");
|
|
23
|
+
await writeFile(filePath, "file content");
|
|
24
|
+
const fileUrl = await path.toFileUrl(filePath);
|
|
25
|
+
const response = await fetchWithFileSupport(fileUrl + "?query=1");
|
|
26
|
+
assert.equal(response.status, 200);
|
|
27
|
+
const content = new TextDecoder().decode(await response.arrayBuffer());
|
|
28
|
+
assert.equal(content, "file content");
|
|
29
|
+
} finally {
|
|
30
|
+
await rm(directory, { recursive: true });
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("fetch returns 404 for a missing file URL", async () => {
|
|
35
|
+
const response = await fetchWithFileSupport("file:///nonexistent/path/resource");
|
|
36
|
+
assert.equal(response.status, 404);
|
|
37
|
+
});
|