single-file-cli 2.0.83 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +30 -0
- package/build.sh +1 -1
- package/compile.sh +18 -13
- package/deno.json +2 -3
- package/deno.lock +545 -56
- package/lib/browser.js +58 -15
- package/lib/cdp-client-util.js +6 -6
- package/lib/cdp-client.js +151 -94
- package/lib/deno-polyfill.js +25 -30
- package/lib/single-file-bundle.js +1 -1
- package/lib/single-file-script.js +3 -1
- package/lib/version.js +1 -1
- package/options.js +2 -1
- package/package.json +13 -8
- package/single-file-cli-api.js +48 -26
- package/single-file-launcher.js +11 -2
- package/test/e2e/crawl.test.js +144 -0
- package/test/e2e/frame-gate.test.js +62 -0
- package/test/e2e/output-json.test.js +45 -0
- package/test/unit/command.test.js +41 -0
- package/test/unit/file-url.test.js +37 -0
package/lib/browser.js
CHANGED
|
@@ -21,18 +21,20 @@
|
|
|
21
21
|
* Source.
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
|
-
/* global fetch */
|
|
24
|
+
/* global fetch, setTimeout */
|
|
25
25
|
|
|
26
26
|
import { BROWSER_PATHS, BROWSER_ARGS } from "./constants.js";
|
|
27
27
|
import { Deno } from "./deno-polyfill.js";
|
|
28
28
|
|
|
29
|
-
const
|
|
29
|
+
const NULL_STD_CONFIG = "null";
|
|
30
30
|
const DEBUG_PORT_MIN = 9222;
|
|
31
31
|
const DEBUG_PORT_RANGE = 256;
|
|
32
|
+
const READY_TIMEOUT = 30000;
|
|
33
|
+
const READY_RETRY_DELAY = 250;
|
|
32
34
|
|
|
33
35
|
const { build, makeTempDir, Command, errors, remove } = Deno;
|
|
34
|
-
let child, profilePath;
|
|
35
|
-
export { launchBrowser, closeBrowser };
|
|
36
|
+
let child, profilePath, childExited;
|
|
37
|
+
export { launchBrowser, closeBrowser, browserExited };
|
|
36
38
|
|
|
37
39
|
async function launchBrowser(options = {}, indexPath = 0) {
|
|
38
40
|
const executablePath = options.executablePath || BROWSER_PATHS[build.os][indexPath];
|
|
@@ -58,7 +60,9 @@ async function launchBrowser(options = {}, indexPath = 0) {
|
|
|
58
60
|
args.push("--proxy-server=" + options.httpProxyServer);
|
|
59
61
|
}
|
|
60
62
|
args.push("--user-data-dir=" + profilePath);
|
|
61
|
-
|
|
63
|
+
if (options.singleProcess) {
|
|
64
|
+
args.push("--single-process");
|
|
65
|
+
}
|
|
62
66
|
if (options.args) {
|
|
63
67
|
const argNames = options.args.map(arg => arg.split("=")[0]);
|
|
64
68
|
args = args.filter(arg => !argNames.includes(arg.split("=")[0]));
|
|
@@ -70,24 +74,59 @@ async function launchBrowser(options = {}, indexPath = 0) {
|
|
|
70
74
|
!args.includes("--headless")) {
|
|
71
75
|
args.push("--disable-site-isolation-trials");
|
|
72
76
|
}
|
|
73
|
-
const command = new Command(executablePath, { args, stdout:
|
|
77
|
+
const command = new Command(executablePath, { args, stdout: NULL_STD_CONFIG, stderr: NULL_STD_CONFIG });
|
|
74
78
|
try {
|
|
75
79
|
child = await command.spawn();
|
|
76
80
|
} catch (error) {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
throw error;
|
|
82
|
-
}
|
|
83
|
-
} else {
|
|
84
|
-
throw error;
|
|
81
|
+
await remove(profilePath, { recursive: true }).catch(() => { });
|
|
82
|
+
profilePath = undefined;
|
|
83
|
+
if (error instanceof errors.NotFound && indexPath + 1 < BROWSER_PATHS[build.os].length) {
|
|
84
|
+
return launchBrowser(options, indexPath + 1);
|
|
85
85
|
}
|
|
86
|
+
throw error;
|
|
86
87
|
}
|
|
87
88
|
child.ref();
|
|
89
|
+
childExited = false;
|
|
90
|
+
child.status.then(() => childExited = true);
|
|
91
|
+
try {
|
|
92
|
+
await waitUntilReady(debugPort);
|
|
93
|
+
} catch (error) {
|
|
94
|
+
await closeBrowser();
|
|
95
|
+
if (options.singleProcess) {
|
|
96
|
+
console.warn("Warning: the browser exited when using --browser-single-process, retrying without it"); // eslint-disable-line no-console
|
|
97
|
+
return launchBrowser(Object.assign({}, options, { singleProcess: false }), indexPath);
|
|
98
|
+
}
|
|
99
|
+
throw error;
|
|
100
|
+
}
|
|
88
101
|
return debugPort;
|
|
89
102
|
}
|
|
90
103
|
|
|
104
|
+
async function waitUntilReady(debugPort) {
|
|
105
|
+
const timeoutTime = Date.now() + READY_TIMEOUT;
|
|
106
|
+
while (!childExited && Date.now() < timeoutTime) {
|
|
107
|
+
try {
|
|
108
|
+
await fetch("http://localhost:" + debugPort + "/json/version");
|
|
109
|
+
return;
|
|
110
|
+
} catch {
|
|
111
|
+
await new Promise(resolve => setTimeout(resolve, READY_RETRY_DELAY));
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
throw new Error(childExited ? "The browser exited unexpectedly" : "The browser is not responding");
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
async function browserExited(maxDelay = 0) {
|
|
118
|
+
if (child === undefined) {
|
|
119
|
+
return false;
|
|
120
|
+
}
|
|
121
|
+
if (childExited || !maxDelay) {
|
|
122
|
+
return childExited;
|
|
123
|
+
}
|
|
124
|
+
return Promise.race([
|
|
125
|
+
child.status.then(() => true),
|
|
126
|
+
new Promise(resolve => setTimeout(() => resolve(childExited), maxDelay))
|
|
127
|
+
]);
|
|
128
|
+
}
|
|
129
|
+
|
|
91
130
|
async function getDebugPort(port = getRandomDebugPort(), usedPorts = []) {
|
|
92
131
|
try {
|
|
93
132
|
await fetch("http://localhost:" + port + "/json/version");
|
|
@@ -112,7 +151,11 @@ function getRandomDebugPort() {
|
|
|
112
151
|
|
|
113
152
|
async function closeBrowser() {
|
|
114
153
|
if (child !== undefined) {
|
|
115
|
-
|
|
154
|
+
try {
|
|
155
|
+
child.kill();
|
|
156
|
+
} catch {
|
|
157
|
+
// ignored
|
|
158
|
+
}
|
|
116
159
|
await child.status;
|
|
117
160
|
child = undefined;
|
|
118
161
|
}
|
package/lib/cdp-client-util.js
CHANGED
|
@@ -21,9 +21,10 @@
|
|
|
21
21
|
* Source.
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
|
-
/* global setTimeout, clearTimeout, fetch, URL, Headers
|
|
24
|
+
/* global setTimeout, clearTimeout, fetch, URL, Headers */
|
|
25
25
|
|
|
26
|
-
import {
|
|
26
|
+
import { Buffer } from "node:buffer";
|
|
27
|
+
import { Deno, isDeno, path } from "./deno-polyfill.js";
|
|
27
28
|
|
|
28
29
|
const ABORT_EVENT = "abort";
|
|
29
30
|
|
|
@@ -42,10 +43,9 @@ async function fetchWithFileSupport(url, fetchOptions = {}) {
|
|
|
42
43
|
if (isDeno) {
|
|
43
44
|
return await fetch(url, fetchOptions);
|
|
44
45
|
}
|
|
45
|
-
const filePath = decodeURIComponent(url.replace(/^file:\/\//, ""));
|
|
46
46
|
const { readFile } = Deno;
|
|
47
47
|
try {
|
|
48
|
-
const fileData = await readFile(
|
|
48
|
+
const fileData = await readFile(await path.fromFileUrl(url));
|
|
49
49
|
return createFileResponse(fileData, 200);
|
|
50
50
|
} catch {
|
|
51
51
|
return createFileResponse(new ArrayBuffer(0), 404);
|
|
@@ -59,7 +59,7 @@ async function fetchWithFileSupport(url, fetchOptions = {}) {
|
|
|
59
59
|
"content-type": isError ? "text/plain" : "application/octet-stream",
|
|
60
60
|
"content-length": data.length ? data.length.toString() : "0"
|
|
61
61
|
}),
|
|
62
|
-
arrayBuffer:
|
|
62
|
+
arrayBuffer: () => data.buffer || data
|
|
63
63
|
};
|
|
64
64
|
}
|
|
65
65
|
}
|
|
@@ -83,7 +83,7 @@ function waitForTimeout(abortSignal, maxDelay, errorMessage, errorCode) {
|
|
|
83
83
|
}
|
|
84
84
|
|
|
85
85
|
function arrayBufferToBase64(arrayBuffer) {
|
|
86
|
-
return
|
|
86
|
+
return Buffer.from(arrayBuffer).toString("base64");
|
|
87
87
|
}
|
|
88
88
|
|
|
89
89
|
function getAlternativeUrl(url) {
|
package/lib/cdp-client.js
CHANGED
|
@@ -25,7 +25,8 @@
|
|
|
25
25
|
|
|
26
26
|
import {
|
|
27
27
|
launchBrowser,
|
|
28
|
-
closeBrowser
|
|
28
|
+
closeBrowser,
|
|
29
|
+
browserExited
|
|
29
30
|
} from "./browser.js";
|
|
30
31
|
import {
|
|
31
32
|
CDP,
|
|
@@ -57,6 +58,10 @@ const SET_SCREENSHOT_FUNCTION_NAME = "setScreenshot";
|
|
|
57
58
|
const SET_PDF_FUNCTION_NAME = "setPDF";
|
|
58
59
|
const SET_PAGE_DATA_FUNCTION_NAME = "setPageData";
|
|
59
60
|
const BINDING_CALLED_EVENT_TYPE = "bindingCalled";
|
|
61
|
+
const LOCALHOST = "http://localhost:";
|
|
62
|
+
const BROWSER_EXITED_MAX_DELAY = 2000;
|
|
63
|
+
|
|
64
|
+
let browserOptions, relaunchBrowserPromise;
|
|
60
65
|
|
|
61
66
|
export {
|
|
62
67
|
initialize,
|
|
@@ -68,12 +73,12 @@ async function initialize(singleFileOptions) {
|
|
|
68
73
|
if (singleFileOptions.browserServer) {
|
|
69
74
|
options.apiUrl = singleFileOptions.browserServer;
|
|
70
75
|
} else {
|
|
71
|
-
|
|
72
|
-
const browserOptions = {};
|
|
76
|
+
browserOptions = {};
|
|
73
77
|
browserOptions.args = singleFileOptions.browserArgs;
|
|
74
78
|
browserOptions.headless = singleFileOptions.browserHeadless;
|
|
75
79
|
browserOptions.executablePath = singleFileOptions.browserExecutablePath;
|
|
76
80
|
browserOptions.debug = singleFileOptions.browserDebug;
|
|
81
|
+
browserOptions.singleProcess = singleFileOptions.browserSingleProcess;
|
|
77
82
|
browserOptions.disableWebSecurity = singleFileOptions.browserDisableWebSecurity;
|
|
78
83
|
browserOptions.width = singleFileOptions.browserWidth;
|
|
79
84
|
browserOptions.height = singleFileOptions.browserHeight;
|
|
@@ -83,14 +88,26 @@ async function initialize(singleFileOptions) {
|
|
|
83
88
|
}
|
|
84
89
|
}
|
|
85
90
|
|
|
91
|
+
function relaunchBrowser() {
|
|
92
|
+
if (!relaunchBrowserPromise) {
|
|
93
|
+
console.warn("Warning: the browser exited when using --browser-single-process, retrying without it"); // eslint-disable-line no-console
|
|
94
|
+
relaunchBrowserPromise = (async () => {
|
|
95
|
+
await closeBrowser();
|
|
96
|
+
browserOptions.singleProcess = false;
|
|
97
|
+
options.apiUrl = LOCALHOST + (await launchBrowser(browserOptions));
|
|
98
|
+
})();
|
|
99
|
+
}
|
|
100
|
+
return relaunchBrowserPromise;
|
|
101
|
+
}
|
|
102
|
+
|
|
86
103
|
async function getPageData(options) {
|
|
87
104
|
const EMPTY_PAGE_URL = "about:blank";
|
|
88
105
|
const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {} };
|
|
89
|
-
let targetInfo;
|
|
106
|
+
let targetInfo, cdp;
|
|
90
107
|
try {
|
|
91
108
|
logData(["Loading page", EMPTY_PAGE_URL], pageContext);
|
|
92
109
|
targetInfo = await CDP.createTarget(EMPTY_PAGE_URL);
|
|
93
|
-
|
|
110
|
+
cdp = new CDP(targetInfo);
|
|
94
111
|
await setupConsoleLogging(cdp, pageContext);
|
|
95
112
|
await setupBrowserWindow(cdp, targetInfo.id, pageContext);
|
|
96
113
|
await setupSecurity(cdp, pageContext);
|
|
@@ -107,6 +124,9 @@ async function getPageData(options) {
|
|
|
107
124
|
if (shouldRetryWithFallback(error)) {
|
|
108
125
|
return await retryWithFallback();
|
|
109
126
|
}
|
|
127
|
+
if (await shouldRelaunchBrowser()) {
|
|
128
|
+
return await relaunchBrowserAndRetry();
|
|
129
|
+
}
|
|
110
130
|
attachDebugInfo(error, pageContext);
|
|
111
131
|
throw error;
|
|
112
132
|
} finally {
|
|
@@ -130,11 +150,36 @@ async function getPageData(options) {
|
|
|
130
150
|
return await getPageData(options);
|
|
131
151
|
}
|
|
132
152
|
|
|
153
|
+
async function shouldRelaunchBrowser() {
|
|
154
|
+
return Boolean(browserOptions && browserOptions.singleProcess) && await browserExited(BROWSER_EXITED_MAX_DELAY);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
async function relaunchBrowserAndRetry() {
|
|
158
|
+
logData(["Relaunching the browser"], pageContext);
|
|
159
|
+
targetInfo = null;
|
|
160
|
+
if (cdp) {
|
|
161
|
+
cdp.reset();
|
|
162
|
+
cdp = null;
|
|
163
|
+
}
|
|
164
|
+
await relaunchBrowser();
|
|
165
|
+
return await getPageData(options);
|
|
166
|
+
}
|
|
167
|
+
|
|
133
168
|
async function closeTarget() {
|
|
134
169
|
if (targetInfo && !options.browserDebug) {
|
|
135
|
-
|
|
170
|
+
try {
|
|
171
|
+
await CDP.closeTarget(targetInfo.id);
|
|
172
|
+
} catch {
|
|
173
|
+
// ignored
|
|
174
|
+
}
|
|
136
175
|
targetInfo = null;
|
|
137
176
|
}
|
|
177
|
+
// the connection is closed even when the target is left open for
|
|
178
|
+
// debugging, otherwise it stays open until the process exits
|
|
179
|
+
if (cdp) {
|
|
180
|
+
cdp.reset();
|
|
181
|
+
cdp = null;
|
|
182
|
+
}
|
|
138
183
|
}
|
|
139
184
|
}
|
|
140
185
|
|
|
@@ -243,7 +288,7 @@ async function setupNetworkInterception({ Browser, Emulation, Fetch, Network },
|
|
|
243
288
|
function setupProxyAuth({ Fetch }, { options, debugMessages }) {
|
|
244
289
|
const AUTH_REQUIRED_EVENT_TYPE = "authRequired";
|
|
245
290
|
const PROVIDE_CREDENTIALS_RESPONSE = "ProvideCredentials";
|
|
246
|
-
Fetch.addEventListener(AUTH_REQUIRED_EVENT_TYPE, async ({ params }) => {
|
|
291
|
+
Fetch.addEventListener(AUTH_REQUIRED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
|
|
247
292
|
logData(["Authenticating"], { options, debugMessages });
|
|
248
293
|
await Fetch.continueWithAuth({
|
|
249
294
|
requestId: params.requestId,
|
|
@@ -253,17 +298,28 @@ function setupProxyAuth({ Fetch }, { options, debugMessages }) {
|
|
|
253
298
|
password: options.httpProxyPassword
|
|
254
299
|
}
|
|
255
300
|
});
|
|
256
|
-
});
|
|
301
|
+
}, { options, debugMessages }));
|
|
257
302
|
}
|
|
258
303
|
|
|
259
304
|
function setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo }) {
|
|
260
305
|
const REQUEST_PAUSED_EVENT_TYPE = "requestPaused";
|
|
261
306
|
const ABORTED_ERROR_REASON = "Aborted";
|
|
262
307
|
const urlState = { url: options.url, alternativeUrl: getAlternativeUrl(options.url) };
|
|
263
|
-
|
|
308
|
+
// compiled here so that an invalid pattern is reported before the page is
|
|
309
|
+
// loaded, instead of throwing for every request that is intercepted
|
|
310
|
+
const blockedURLPatterns = (options.blockedURLPatterns || []).map(pattern => new RegExp(pattern));
|
|
311
|
+
Fetch.addEventListener(REQUEST_PAUSED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
|
|
264
312
|
const { requestId, request } = params;
|
|
265
|
-
|
|
266
|
-
|
|
313
|
+
// the request is always resumed below, otherwise the page waits for it
|
|
314
|
+
// until the load timeout expires
|
|
315
|
+
let blocked = false;
|
|
316
|
+
try {
|
|
317
|
+
captureHttpInfo(params, urlState, { options, debugMessages, httpInfo });
|
|
318
|
+
blocked = shouldBlockRequest(request.url);
|
|
319
|
+
} catch (error) {
|
|
320
|
+
logData(["Ignoring request interception error", error.message], { options, debugMessages });
|
|
321
|
+
}
|
|
322
|
+
if (blocked) {
|
|
267
323
|
try {
|
|
268
324
|
await Fetch.failRequest({ requestId, errorReason: ABORTED_ERROR_REASON });
|
|
269
325
|
return;
|
|
@@ -276,15 +332,10 @@ function setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo
|
|
|
276
332
|
} catch {
|
|
277
333
|
// ignored
|
|
278
334
|
}
|
|
279
|
-
});
|
|
335
|
+
}, { options, debugMessages }));
|
|
280
336
|
|
|
281
337
|
function shouldBlockRequest(requestUrl) {
|
|
282
|
-
|
|
283
|
-
return false;
|
|
284
|
-
}
|
|
285
|
-
const blockedURL = options.blockedURLPatterns.find(pattern =>
|
|
286
|
-
new RegExp(pattern).test(requestUrl)
|
|
287
|
-
);
|
|
338
|
+
const blockedURL = blockedURLPatterns.some(pattern => pattern.test(requestUrl));
|
|
288
339
|
if (blockedURL) {
|
|
289
340
|
logData(["Blocking request", requestUrl], { options, debugMessages });
|
|
290
341
|
return true;
|
|
@@ -336,13 +387,14 @@ async function setupHttpHeaders({ Network }, { options, debugMessages }) {
|
|
|
336
387
|
}
|
|
337
388
|
|
|
338
389
|
async function setupMediaFeatures({ Emulation }, { options, debugMessages }) {
|
|
390
|
+
const features = [];
|
|
339
391
|
for (const mediaFeature of options.emulateMediaFeatures) {
|
|
340
392
|
logData(["Emulating media feature", mediaFeature.name, mediaFeature.value], { options, debugMessages });
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
});
|
|
393
|
+
for (const value of mediaFeature.value.split(",")) {
|
|
394
|
+
features.push({ name: mediaFeature.name, value: value.trim() });
|
|
395
|
+
}
|
|
345
396
|
}
|
|
397
|
+
await Emulation.setEmulatedMedia({ features });
|
|
346
398
|
}
|
|
347
399
|
|
|
348
400
|
async function setupCookies({ Network }, { options, debugMessages }) {
|
|
@@ -387,12 +439,19 @@ async function loadPage({ Page, Runtime }, { options, debugMessages }) {
|
|
|
387
439
|
const LOAD_TIMEOUT_ERROR_MESSAGE = "Load timeout";
|
|
388
440
|
await Runtime.enable();
|
|
389
441
|
await Page.enable();
|
|
442
|
+
await Page.setLifecycleEventsEnabled({ enabled: true });
|
|
443
|
+
// the ID of the top frame is stable, and it is read before the navigation is
|
|
444
|
+
// triggered so that no event is missed while the browser answers
|
|
445
|
+
const { frameTree } = await Page.getFrameTree();
|
|
390
446
|
const loadTimeoutAbortController = new AbortController();
|
|
391
447
|
const loadTimeoutAbortSignal = loadTimeoutAbortController.signal;
|
|
392
448
|
try {
|
|
393
449
|
logData(["Loading page", options.url], { options, debugMessages });
|
|
394
450
|
const [contextId] = await Promise.race([
|
|
395
|
-
Promise.all([
|
|
451
|
+
Promise.all([
|
|
452
|
+
getTopFrameContextId({ Page, Runtime }, frameTree.frame.id, { options, debugMessages }),
|
|
453
|
+
Page.navigate({ url: options.url })
|
|
454
|
+
]),
|
|
396
455
|
waitForTimeout(loadTimeoutAbortSignal, options.browserLoadMaxTime, LOAD_TIMEOUT_ERROR_MESSAGE, LOAD_TIMEOUT_ERROR)
|
|
397
456
|
]);
|
|
398
457
|
return contextId;
|
|
@@ -400,35 +459,17 @@ async function loadPage({ Page, Runtime }, { options, debugMessages }) {
|
|
|
400
459
|
if (!loadTimeoutAbortSignal.aborted) {
|
|
401
460
|
loadTimeoutAbortController.abort();
|
|
402
461
|
}
|
|
462
|
+
await Page.setLifecycleEventsEnabled({ enabled: false });
|
|
403
463
|
await Runtime.disable();
|
|
404
464
|
await Page.disable();
|
|
405
465
|
}
|
|
406
466
|
}
|
|
407
467
|
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
await waitForPageReadyState({ Page }, state, { options, debugMessages });
|
|
414
|
-
const contextId = await findValidSingleFileContext({ Runtime }, state.contextIds, { options, debugMessages });
|
|
415
|
-
return contextId;
|
|
416
|
-
} finally {
|
|
417
|
-
removeContextListener();
|
|
418
|
-
await Page.setLifecycleEventsEnabled({ enabled: false });
|
|
419
|
-
}
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
function setupContextCreatedListener({ Runtime }, state) {
|
|
423
|
-
const EXECUTION_CONTEXT_CREATED_EVENT_TYPE = "executionContextCreated";
|
|
424
|
-
const onContextCreated = ({ params }) => {
|
|
425
|
-
const { context } = params;
|
|
426
|
-
if (context.name === SINGLE_FILE_WORLD_NAME && context.auxData?.frameId === state.topFrameId) {
|
|
427
|
-
state.contextIds.push(context.id);
|
|
428
|
-
}
|
|
429
|
-
};
|
|
430
|
-
Runtime.addEventListener(EXECUTION_CONTEXT_CREATED_EVENT_TYPE, onContextCreated);
|
|
431
|
-
return () => Runtime.removeEventListener(EXECUTION_CONTEXT_CREATED_EVENT_TYPE, onContextCreated);
|
|
468
|
+
// the listeners are registered synchronously, before the navigation triggered
|
|
469
|
+
// in parallel by the caller can produce any event
|
|
470
|
+
async function getTopFrameContextId({ Page, Runtime }, topFrameId, { options, debugMessages }) {
|
|
471
|
+
await waitForPageReadyState({ Page }, { topFrameId }, { options, debugMessages });
|
|
472
|
+
return await getSingleFileContext({ Page, Runtime }, topFrameId, { options, debugMessages });
|
|
432
473
|
}
|
|
433
474
|
|
|
434
475
|
async function waitForPageReadyState({ Page }, state, { options, debugMessages }) {
|
|
@@ -441,7 +482,7 @@ async function waitForPageReadyState({ Page }, state, { options, debugMessages }
|
|
|
441
482
|
Page.removeEventListener(FRAME_NAVIGATED_EVENT_TYPE, onFrameNavigated);
|
|
442
483
|
};
|
|
443
484
|
const onLifecycleEvent = createLifecycleEventHandler(state, timeoutState, resolve, cleanup, { options, debugMessages });
|
|
444
|
-
const onFrameNavigated = createFrameNavigatedHandler(
|
|
485
|
+
const onFrameNavigated = createFrameNavigatedHandler(timeoutState, reject, cleanup, { options, debugMessages });
|
|
445
486
|
Page.addEventListener(LIFE_CYCLE_EVENT_TYPE, onLifecycleEvent);
|
|
446
487
|
Page.addEventListener(FRAME_NAVIGATED_EVENT_TYPE, onFrameNavigated);
|
|
447
488
|
});
|
|
@@ -453,12 +494,15 @@ function createLifecycleEventHandler(state, timeoutState, resolve, cleanup, { op
|
|
|
453
494
|
if (frameId === state.topFrameId) {
|
|
454
495
|
logData(["Detecting lifecycle event", name], { options, debugMessages });
|
|
455
496
|
}
|
|
456
|
-
const shouldResolve =
|
|
457
|
-
(
|
|
497
|
+
const shouldResolve = frameId === state.topFrameId &&
|
|
498
|
+
(name === options.browserWaitUntil ||
|
|
499
|
+
(timeoutState.timeoutId && NETWORK_STATES.indexOf(name) < NETWORK_STATES.indexOf(options.browserWaitUntil)));
|
|
458
500
|
if (shouldResolve) {
|
|
501
|
+
// the delay is restarted when the page reaches a further state, so
|
|
502
|
+
// that it is captured once it stopped settling
|
|
459
503
|
clearTimeout(timeoutState.timeoutId);
|
|
460
504
|
logData([`Waiting ${options.browserWaitUntilDelay} ms`], { options, debugMessages });
|
|
461
|
-
setTimeout(() => {
|
|
505
|
+
timeoutState.timeoutId = setTimeout(() => {
|
|
462
506
|
logData(["Detecting page ready"], { options, debugMessages });
|
|
463
507
|
cleanup();
|
|
464
508
|
resolve();
|
|
@@ -467,49 +511,43 @@ function createLifecycleEventHandler(state, timeoutState, resolve, cleanup, { op
|
|
|
467
511
|
};
|
|
468
512
|
};
|
|
469
513
|
|
|
470
|
-
function createFrameNavigatedHandler(
|
|
514
|
+
function createFrameNavigatedHandler(timeoutState, reject, cleanup, { options, debugMessages }) {
|
|
471
515
|
const UNREACHABLE_URL_ERROR_MESSAGE = "Unreachable URL";
|
|
472
516
|
return ({ params }) => {
|
|
473
517
|
const { frame } = params;
|
|
474
|
-
if (!frame.parentId) {
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
} else {
|
|
480
|
-
logData(["Detecting top frame ID"], { options, debugMessages });
|
|
481
|
-
state.topFrameId = frame.id;
|
|
482
|
-
}
|
|
518
|
+
if (!frame.parentId && frame.unreachableUrl) {
|
|
519
|
+
logData(["Detecting unreachable URL", frame.unreachableUrl], { options, debugMessages });
|
|
520
|
+
clearTimeout(timeoutState.timeoutId);
|
|
521
|
+
cleanup();
|
|
522
|
+
reject(new Error(UNREACHABLE_URL_ERROR_MESSAGE + ": " + frame.unreachableUrl));
|
|
483
523
|
}
|
|
484
524
|
};
|
|
485
525
|
}
|
|
486
526
|
|
|
487
|
-
async function
|
|
488
|
-
const CONTEXT_NOT_FOUND_ERROR_MESSAGE = "Execution context not found for SingleFile world";
|
|
527
|
+
async function getSingleFileContext({ Page, Runtime }, topFrameId, { options, debugMessages }) {
|
|
489
528
|
const SINGLE_FILE_DETECTION_TEST = "typeof singlefile !== 'undefined'";
|
|
490
529
|
const NO_VALID_CONTEXT_ERROR_MESSAGE = "No valid SingleFile execution context found";
|
|
491
530
|
logData(["Getting execution context"], { options, debugMessages });
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
}
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
}
|
|
531
|
+
// the world already exists, so asking for it by name returns the context the
|
|
532
|
+
// injected script ran in instead of creating another one
|
|
533
|
+
const { executionContextId } = await Page.createIsolatedWorld({
|
|
534
|
+
frameId: topFrameId,
|
|
535
|
+
worldName: SINGLE_FILE_WORLD_NAME
|
|
536
|
+
});
|
|
537
|
+
// an empty world is returned when the script could not be injected, so the
|
|
538
|
+
// context is checked before it is used to capture the page
|
|
539
|
+
const { result } = await Runtime.evaluate({
|
|
540
|
+
expression: SINGLE_FILE_DETECTION_TEST,
|
|
541
|
+
contextId: executionContextId
|
|
542
|
+
});
|
|
543
|
+
if (result.value !== true) {
|
|
544
|
+
throw new Error(NO_VALID_CONTEXT_ERROR_MESSAGE);
|
|
507
545
|
}
|
|
508
|
-
|
|
546
|
+
return executionContextId;
|
|
509
547
|
}
|
|
510
548
|
|
|
511
|
-
function setupPageDataCapture({ Runtime },
|
|
512
|
-
return new Promise(resolve => {
|
|
549
|
+
function setupPageDataCapture({ Runtime }, _contextId, { options, debugMessages }) {
|
|
550
|
+
return new Promise((resolve, reject) => {
|
|
513
551
|
let pageDataResponse = "";
|
|
514
552
|
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ({ params }) => {
|
|
515
553
|
if (params.name === SET_PAGE_DATA_FUNCTION_NAME) {
|
|
@@ -518,11 +556,15 @@ function setupPageDataCapture({ Runtime }, contextId, { options, debugMessages }
|
|
|
518
556
|
pageDataResponse += payload;
|
|
519
557
|
} else {
|
|
520
558
|
logData(["Setting page data"], { options, debugMessages });
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
result.content
|
|
559
|
+
try {
|
|
560
|
+
const result = JSON.parse(pageDataResponse);
|
|
561
|
+
if (result.content instanceof Array) {
|
|
562
|
+
result.content = new Uint8Array(result.content);
|
|
563
|
+
}
|
|
564
|
+
resolve(result);
|
|
565
|
+
} catch (error) {
|
|
566
|
+
reject(error);
|
|
524
567
|
}
|
|
525
|
-
resolve(result);
|
|
526
568
|
}
|
|
527
569
|
}
|
|
528
570
|
});
|
|
@@ -534,20 +576,20 @@ async function setupBindings({ Page, Runtime }, contextId, { options, debugMessa
|
|
|
534
576
|
if (options.embedScreenshot && options.compressContent) {
|
|
535
577
|
await setupScreenshotCapture({ Page, Runtime }, contextId, { options, debugMessages });
|
|
536
578
|
}
|
|
537
|
-
if (options.embedPdf) {
|
|
579
|
+
if (options.embedPdf && options.compressContent) {
|
|
538
580
|
await setupPdfCapture({ Page, Runtime }, contextId, { options, debugMessages });
|
|
539
581
|
}
|
|
540
582
|
await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextId: contextId });
|
|
541
|
-
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, async ({ params }) => {
|
|
583
|
+
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
|
|
542
584
|
if (params.name === FETCH_FUNCTION_NAME) {
|
|
543
585
|
await handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages });
|
|
544
586
|
}
|
|
545
|
-
});
|
|
587
|
+
}, { options, debugMessages }));
|
|
546
588
|
}
|
|
547
589
|
|
|
548
590
|
async function setupScreenshotCapture({ Page, Runtime }, contextId, { options, debugMessages }) {
|
|
549
591
|
await Runtime.addBinding({ name: CAPTURE_SCREENSHOT_FUNCTION_NAME, executionContextId: contextId });
|
|
550
|
-
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, async ({ params }) => {
|
|
592
|
+
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
|
|
551
593
|
if (params.name === CAPTURE_SCREENSHOT_FUNCTION_NAME) {
|
|
552
594
|
logData(["Capturing screenshot"], { options, debugMessages });
|
|
553
595
|
try {
|
|
@@ -558,7 +600,7 @@ async function setupScreenshotCapture({ Page, Runtime }, contextId, { options, d
|
|
|
558
600
|
await callBrowserFunction({ Runtime }, contextId, SET_SCREENSHOT_FUNCTION_NAME, [""]);
|
|
559
601
|
}
|
|
560
602
|
}
|
|
561
|
-
});
|
|
603
|
+
}, { options, debugMessages }));
|
|
562
604
|
|
|
563
605
|
function parseScreenshotOptions(optionsString) {
|
|
564
606
|
const PNG_FORMAT = "png";
|
|
@@ -577,8 +619,8 @@ async function setupScreenshotCapture({ Page, Runtime }, contextId, { options, d
|
|
|
577
619
|
|
|
578
620
|
async function setupPdfCapture({ Page, Runtime }, contextId, { options, debugMessages }) {
|
|
579
621
|
await Runtime.addBinding({ name: PRINT_TO_PDF_FUNCTION_NAME, executionContextId: contextId });
|
|
580
|
-
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, async ({ params }) => {
|
|
581
|
-
if (params.name
|
|
622
|
+
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
|
|
623
|
+
if (params.name === PRINT_TO_PDF_FUNCTION_NAME) {
|
|
582
624
|
logData(["Printing to PDF", options.embedPdfOptions || ""], { options, debugMessages });
|
|
583
625
|
const pdfOptions = parsePdfOptions(options.embedPdfOptions);
|
|
584
626
|
try {
|
|
@@ -588,7 +630,7 @@ async function setupPdfCapture({ Page, Runtime }, contextId, { options, debugMes
|
|
|
588
630
|
await callBrowserFunction({ Runtime }, contextId, SET_PDF_FUNCTION_NAME, [""]);
|
|
589
631
|
}
|
|
590
632
|
}
|
|
591
|
-
});
|
|
633
|
+
}, { options, debugMessages }));
|
|
592
634
|
|
|
593
635
|
function parsePdfOptions(optionsString) {
|
|
594
636
|
let pdfOptions = {};
|
|
@@ -671,7 +713,10 @@ async function capturePageData({ Runtime }, contextId, { options, debugMessages
|
|
|
671
713
|
}
|
|
672
714
|
}
|
|
673
715
|
|
|
674
|
-
async function disableCdpDomains({ Console, Network, Page, Runtime }, { options }) {
|
|
716
|
+
async function disableCdpDomains({ Console, Fetch, Network, Page, Runtime }, { options }) {
|
|
717
|
+
// disabled first, so that the requests left paused are resumed by the
|
|
718
|
+
// browser instead of being held until the target is closed
|
|
719
|
+
await Fetch.disable();
|
|
675
720
|
await Runtime.disable();
|
|
676
721
|
await Page.disable();
|
|
677
722
|
if (options.httpHeaders) {
|
|
@@ -708,6 +753,18 @@ function attachDebugInfo(error, { options, consoleMessages, debugMessages }) {
|
|
|
708
753
|
}
|
|
709
754
|
}
|
|
710
755
|
|
|
756
|
+
function ignoringErrors(listener, pageContext) {
|
|
757
|
+
return async event => {
|
|
758
|
+
try {
|
|
759
|
+
await listener(event);
|
|
760
|
+
} catch (error) {
|
|
761
|
+
// the commands sent while the target is closing are rejected, and an
|
|
762
|
+
// error thrown here would be reported as an unhandled rejection
|
|
763
|
+
logData(["Ignoring event listener error", error.message], pageContext);
|
|
764
|
+
}
|
|
765
|
+
};
|
|
766
|
+
}
|
|
767
|
+
|
|
711
768
|
function logData(data, { options, debugMessages }) {
|
|
712
769
|
if (options.debugMessagesFile) {
|
|
713
770
|
debugMessages.push([Date.now(), data]);
|