single-file-cli 2.0.82 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +30 -0
- package/build.sh +1 -1
- package/compile.sh +19 -13
- package/deno.json +2 -3
- package/deno.lock +545 -56
- package/lib/browser.js +10 -11
- package/lib/cdp-client-util.js +5 -6
- package/lib/cdp-client.js +109 -90
- package/lib/deno-polyfill.js +25 -30
- package/lib/single-file-bundle.js +1 -1
- package/lib/single-file-script.js +3 -1
- package/lib/version.js +1 -1
- package/options.js +3 -1
- package/package.json +13 -8
- package/single-file-cli-api.js +47 -26
- package/single-file-launcher.js +11 -2
- package/test/e2e/crawl.test.js +142 -0
- package/test/e2e/frame-gate.test.js +61 -0
- package/test/e2e/output-json.test.js +40 -0
- package/test/unit/command.test.js +41 -0
- package/test/unit/file-url.test.js +37 -0
package/lib/browser.js
CHANGED
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
import { BROWSER_PATHS, BROWSER_ARGS } from "./constants.js";
|
|
27
27
|
import { Deno } from "./deno-polyfill.js";
|
|
28
28
|
|
|
29
|
-
const
|
|
29
|
+
const NULL_STD_CONFIG = "null";
|
|
30
30
|
const DEBUG_PORT_MIN = 9222;
|
|
31
31
|
const DEBUG_PORT_RANGE = 256;
|
|
32
32
|
|
|
@@ -58,7 +58,9 @@ async function launchBrowser(options = {}, indexPath = 0) {
|
|
|
58
58
|
args.push("--proxy-server=" + options.httpProxyServer);
|
|
59
59
|
}
|
|
60
60
|
args.push("--user-data-dir=" + profilePath);
|
|
61
|
-
|
|
61
|
+
if (options.singleProcess) {
|
|
62
|
+
args.push("--single-process");
|
|
63
|
+
}
|
|
62
64
|
if (options.args) {
|
|
63
65
|
const argNames = options.args.map(arg => arg.split("=")[0]);
|
|
64
66
|
args = args.filter(arg => !argNames.includes(arg.split("=")[0]));
|
|
@@ -70,19 +72,16 @@ async function launchBrowser(options = {}, indexPath = 0) {
|
|
|
70
72
|
!args.includes("--headless")) {
|
|
71
73
|
args.push("--disable-site-isolation-trials");
|
|
72
74
|
}
|
|
73
|
-
const command = new Command(executablePath, { args, stdout:
|
|
75
|
+
const command = new Command(executablePath, { args, stdout: NULL_STD_CONFIG, stderr: NULL_STD_CONFIG });
|
|
74
76
|
try {
|
|
75
77
|
child = await command.spawn();
|
|
76
78
|
} catch (error) {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
throw error;
|
|
82
|
-
}
|
|
83
|
-
} else {
|
|
84
|
-
throw error;
|
|
79
|
+
await remove(profilePath, { recursive: true }).catch(() => { });
|
|
80
|
+
profilePath = undefined;
|
|
81
|
+
if (error instanceof errors.NotFound && indexPath + 1 < BROWSER_PATHS[build.os].length) {
|
|
82
|
+
return launchBrowser(options, indexPath + 1);
|
|
85
83
|
}
|
|
84
|
+
throw error;
|
|
86
85
|
}
|
|
87
86
|
child.ref();
|
|
88
87
|
return debugPort;
|
package/lib/cdp-client-util.js
CHANGED
|
@@ -21,9 +21,9 @@
|
|
|
21
21
|
* Source.
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
|
-
/* global setTimeout, clearTimeout, fetch, URL, Headers
|
|
24
|
+
/* global setTimeout, clearTimeout, fetch, URL, Headers */
|
|
25
25
|
|
|
26
|
-
import { Deno, isDeno } from "./deno-polyfill.js";
|
|
26
|
+
import { Deno, isDeno, path } from "./deno-polyfill.js";
|
|
27
27
|
|
|
28
28
|
const ABORT_EVENT = "abort";
|
|
29
29
|
|
|
@@ -42,10 +42,9 @@ async function fetchWithFileSupport(url, fetchOptions = {}) {
|
|
|
42
42
|
if (isDeno) {
|
|
43
43
|
return await fetch(url, fetchOptions);
|
|
44
44
|
}
|
|
45
|
-
const filePath = decodeURIComponent(url.replace(/^file:\/\//, ""));
|
|
46
45
|
const { readFile } = Deno;
|
|
47
46
|
try {
|
|
48
|
-
const fileData = await readFile(
|
|
47
|
+
const fileData = await readFile(await path.fromFileUrl(url));
|
|
49
48
|
return createFileResponse(fileData, 200);
|
|
50
49
|
} catch {
|
|
51
50
|
return createFileResponse(new ArrayBuffer(0), 404);
|
|
@@ -59,7 +58,7 @@ async function fetchWithFileSupport(url, fetchOptions = {}) {
|
|
|
59
58
|
"content-type": isError ? "text/plain" : "application/octet-stream",
|
|
60
59
|
"content-length": data.length ? data.length.toString() : "0"
|
|
61
60
|
}),
|
|
62
|
-
arrayBuffer:
|
|
61
|
+
arrayBuffer: () => data.buffer || data
|
|
63
62
|
};
|
|
64
63
|
}
|
|
65
64
|
}
|
|
@@ -83,7 +82,7 @@ function waitForTimeout(abortSignal, maxDelay, errorMessage, errorCode) {
|
|
|
83
82
|
}
|
|
84
83
|
|
|
85
84
|
function arrayBufferToBase64(arrayBuffer) {
|
|
86
|
-
return
|
|
85
|
+
return new Uint8Array(arrayBuffer).toBase64();
|
|
87
86
|
}
|
|
88
87
|
|
|
89
88
|
function getAlternativeUrl(url) {
|
package/lib/cdp-client.js
CHANGED
|
@@ -74,6 +74,7 @@ async function initialize(singleFileOptions) {
|
|
|
74
74
|
browserOptions.headless = singleFileOptions.browserHeadless;
|
|
75
75
|
browserOptions.executablePath = singleFileOptions.browserExecutablePath;
|
|
76
76
|
browserOptions.debug = singleFileOptions.browserDebug;
|
|
77
|
+
browserOptions.singleProcess = singleFileOptions.browserSingleProcess;
|
|
77
78
|
browserOptions.disableWebSecurity = singleFileOptions.browserDisableWebSecurity;
|
|
78
79
|
browserOptions.width = singleFileOptions.browserWidth;
|
|
79
80
|
browserOptions.height = singleFileOptions.browserHeight;
|
|
@@ -86,11 +87,11 @@ async function initialize(singleFileOptions) {
|
|
|
86
87
|
async function getPageData(options) {
|
|
87
88
|
const EMPTY_PAGE_URL = "about:blank";
|
|
88
89
|
const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {} };
|
|
89
|
-
let targetInfo;
|
|
90
|
+
let targetInfo, cdp;
|
|
90
91
|
try {
|
|
91
92
|
logData(["Loading page", EMPTY_PAGE_URL], pageContext);
|
|
92
93
|
targetInfo = await CDP.createTarget(EMPTY_PAGE_URL);
|
|
93
|
-
|
|
94
|
+
cdp = new CDP(targetInfo);
|
|
94
95
|
await setupConsoleLogging(cdp, pageContext);
|
|
95
96
|
await setupBrowserWindow(cdp, targetInfo.id, pageContext);
|
|
96
97
|
await setupSecurity(cdp, pageContext);
|
|
@@ -135,6 +136,12 @@ async function getPageData(options) {
|
|
|
135
136
|
await CDP.closeTarget(targetInfo.id);
|
|
136
137
|
targetInfo = null;
|
|
137
138
|
}
|
|
139
|
+
// the connection is closed even when the target is left open for
|
|
140
|
+
// debugging, otherwise it stays open until the process exits
|
|
141
|
+
if (cdp) {
|
|
142
|
+
cdp.reset();
|
|
143
|
+
cdp = null;
|
|
144
|
+
}
|
|
138
145
|
}
|
|
139
146
|
}
|
|
140
147
|
|
|
@@ -243,7 +250,7 @@ async function setupNetworkInterception({ Browser, Emulation, Fetch, Network },
|
|
|
243
250
|
function setupProxyAuth({ Fetch }, { options, debugMessages }) {
|
|
244
251
|
const AUTH_REQUIRED_EVENT_TYPE = "authRequired";
|
|
245
252
|
const PROVIDE_CREDENTIALS_RESPONSE = "ProvideCredentials";
|
|
246
|
-
Fetch.addEventListener(AUTH_REQUIRED_EVENT_TYPE, async ({ params }) => {
|
|
253
|
+
Fetch.addEventListener(AUTH_REQUIRED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
|
|
247
254
|
logData(["Authenticating"], { options, debugMessages });
|
|
248
255
|
await Fetch.continueWithAuth({
|
|
249
256
|
requestId: params.requestId,
|
|
@@ -253,17 +260,28 @@ function setupProxyAuth({ Fetch }, { options, debugMessages }) {
|
|
|
253
260
|
password: options.httpProxyPassword
|
|
254
261
|
}
|
|
255
262
|
});
|
|
256
|
-
});
|
|
263
|
+
}, { options, debugMessages }));
|
|
257
264
|
}
|
|
258
265
|
|
|
259
266
|
function setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo }) {
|
|
260
267
|
const REQUEST_PAUSED_EVENT_TYPE = "requestPaused";
|
|
261
268
|
const ABORTED_ERROR_REASON = "Aborted";
|
|
262
269
|
const urlState = { url: options.url, alternativeUrl: getAlternativeUrl(options.url) };
|
|
263
|
-
|
|
270
|
+
// compiled here so that an invalid pattern is reported before the page is
|
|
271
|
+
// loaded, instead of throwing for every request that is intercepted
|
|
272
|
+
const blockedURLPatterns = (options.blockedURLPatterns || []).map(pattern => new RegExp(pattern));
|
|
273
|
+
Fetch.addEventListener(REQUEST_PAUSED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
|
|
264
274
|
const { requestId, request } = params;
|
|
265
|
-
|
|
266
|
-
|
|
275
|
+
// the request is always resumed below, otherwise the page waits for it
|
|
276
|
+
// until the load timeout expires
|
|
277
|
+
let blocked = false;
|
|
278
|
+
try {
|
|
279
|
+
captureHttpInfo(params, urlState, { options, debugMessages, httpInfo });
|
|
280
|
+
blocked = shouldBlockRequest(request.url);
|
|
281
|
+
} catch (error) {
|
|
282
|
+
logData(["Ignoring request interception error", error.message], { options, debugMessages });
|
|
283
|
+
}
|
|
284
|
+
if (blocked) {
|
|
267
285
|
try {
|
|
268
286
|
await Fetch.failRequest({ requestId, errorReason: ABORTED_ERROR_REASON });
|
|
269
287
|
return;
|
|
@@ -276,15 +294,10 @@ function setupRequestInterception({ Fetch }, { options, debugMessages, httpInfo
|
|
|
276
294
|
} catch {
|
|
277
295
|
// ignored
|
|
278
296
|
}
|
|
279
|
-
});
|
|
297
|
+
}, { options, debugMessages }));
|
|
280
298
|
|
|
281
299
|
function shouldBlockRequest(requestUrl) {
|
|
282
|
-
|
|
283
|
-
return false;
|
|
284
|
-
}
|
|
285
|
-
const blockedURL = options.blockedURLPatterns.find(pattern =>
|
|
286
|
-
new RegExp(pattern).test(requestUrl)
|
|
287
|
-
);
|
|
300
|
+
const blockedURL = blockedURLPatterns.some(pattern => pattern.test(requestUrl));
|
|
288
301
|
if (blockedURL) {
|
|
289
302
|
logData(["Blocking request", requestUrl], { options, debugMessages });
|
|
290
303
|
return true;
|
|
@@ -336,13 +349,14 @@ async function setupHttpHeaders({ Network }, { options, debugMessages }) {
|
|
|
336
349
|
}
|
|
337
350
|
|
|
338
351
|
async function setupMediaFeatures({ Emulation }, { options, debugMessages }) {
|
|
352
|
+
const features = [];
|
|
339
353
|
for (const mediaFeature of options.emulateMediaFeatures) {
|
|
340
354
|
logData(["Emulating media feature", mediaFeature.name, mediaFeature.value], { options, debugMessages });
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
});
|
|
355
|
+
for (const value of mediaFeature.value.split(",")) {
|
|
356
|
+
features.push({ name: mediaFeature.name, value: value.trim() });
|
|
357
|
+
}
|
|
345
358
|
}
|
|
359
|
+
await Emulation.setEmulatedMedia({ features });
|
|
346
360
|
}
|
|
347
361
|
|
|
348
362
|
async function setupCookies({ Network }, { options, debugMessages }) {
|
|
@@ -387,12 +401,19 @@ async function loadPage({ Page, Runtime }, { options, debugMessages }) {
|
|
|
387
401
|
const LOAD_TIMEOUT_ERROR_MESSAGE = "Load timeout";
|
|
388
402
|
await Runtime.enable();
|
|
389
403
|
await Page.enable();
|
|
404
|
+
await Page.setLifecycleEventsEnabled({ enabled: true });
|
|
405
|
+
// the ID of the top frame is stable, and it is read before the navigation is
|
|
406
|
+
// triggered so that no event is missed while the browser answers
|
|
407
|
+
const { frameTree } = await Page.getFrameTree();
|
|
390
408
|
const loadTimeoutAbortController = new AbortController();
|
|
391
409
|
const loadTimeoutAbortSignal = loadTimeoutAbortController.signal;
|
|
392
410
|
try {
|
|
393
411
|
logData(["Loading page", options.url], { options, debugMessages });
|
|
394
412
|
const [contextId] = await Promise.race([
|
|
395
|
-
Promise.all([
|
|
413
|
+
Promise.all([
|
|
414
|
+
getTopFrameContextId({ Page, Runtime }, frameTree.frame.id, { options, debugMessages }),
|
|
415
|
+
Page.navigate({ url: options.url })
|
|
416
|
+
]),
|
|
396
417
|
waitForTimeout(loadTimeoutAbortSignal, options.browserLoadMaxTime, LOAD_TIMEOUT_ERROR_MESSAGE, LOAD_TIMEOUT_ERROR)
|
|
397
418
|
]);
|
|
398
419
|
return contextId;
|
|
@@ -400,35 +421,17 @@ async function loadPage({ Page, Runtime }, { options, debugMessages }) {
|
|
|
400
421
|
if (!loadTimeoutAbortSignal.aborted) {
|
|
401
422
|
loadTimeoutAbortController.abort();
|
|
402
423
|
}
|
|
424
|
+
await Page.setLifecycleEventsEnabled({ enabled: false });
|
|
403
425
|
await Runtime.disable();
|
|
404
426
|
await Page.disable();
|
|
405
427
|
}
|
|
406
428
|
}
|
|
407
429
|
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
await waitForPageReadyState({ Page }, state, { options, debugMessages });
|
|
414
|
-
const contextId = await findValidSingleFileContext({ Runtime }, state.contextIds, { options, debugMessages });
|
|
415
|
-
return contextId;
|
|
416
|
-
} finally {
|
|
417
|
-
removeContextListener();
|
|
418
|
-
await Page.setLifecycleEventsEnabled({ enabled: false });
|
|
419
|
-
}
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
function setupContextCreatedListener({ Runtime }, state) {
|
|
423
|
-
const EXECUTION_CONTEXT_CREATED_EVENT_TYPE = "executionContextCreated";
|
|
424
|
-
const onContextCreated = ({ params }) => {
|
|
425
|
-
const { context } = params;
|
|
426
|
-
if (context.name === SINGLE_FILE_WORLD_NAME && context.auxData?.frameId === state.topFrameId) {
|
|
427
|
-
state.contextIds.push(context.id);
|
|
428
|
-
}
|
|
429
|
-
};
|
|
430
|
-
Runtime.addEventListener(EXECUTION_CONTEXT_CREATED_EVENT_TYPE, onContextCreated);
|
|
431
|
-
return () => Runtime.removeEventListener(EXECUTION_CONTEXT_CREATED_EVENT_TYPE, onContextCreated);
|
|
430
|
+
// the listeners are registered synchronously, before the navigation triggered
|
|
431
|
+
// in parallel by the caller can produce any event
|
|
432
|
+
async function getTopFrameContextId({ Page, Runtime }, topFrameId, { options, debugMessages }) {
|
|
433
|
+
await waitForPageReadyState({ Page }, { topFrameId }, { options, debugMessages });
|
|
434
|
+
return await getSingleFileContext({ Page, Runtime }, topFrameId, { options, debugMessages });
|
|
432
435
|
}
|
|
433
436
|
|
|
434
437
|
async function waitForPageReadyState({ Page }, state, { options, debugMessages }) {
|
|
@@ -441,7 +444,7 @@ async function waitForPageReadyState({ Page }, state, { options, debugMessages }
|
|
|
441
444
|
Page.removeEventListener(FRAME_NAVIGATED_EVENT_TYPE, onFrameNavigated);
|
|
442
445
|
};
|
|
443
446
|
const onLifecycleEvent = createLifecycleEventHandler(state, timeoutState, resolve, cleanup, { options, debugMessages });
|
|
444
|
-
const onFrameNavigated = createFrameNavigatedHandler(
|
|
447
|
+
const onFrameNavigated = createFrameNavigatedHandler(timeoutState, reject, cleanup, { options, debugMessages });
|
|
445
448
|
Page.addEventListener(LIFE_CYCLE_EVENT_TYPE, onLifecycleEvent);
|
|
446
449
|
Page.addEventListener(FRAME_NAVIGATED_EVENT_TYPE, onFrameNavigated);
|
|
447
450
|
});
|
|
@@ -453,12 +456,15 @@ function createLifecycleEventHandler(state, timeoutState, resolve, cleanup, { op
|
|
|
453
456
|
if (frameId === state.topFrameId) {
|
|
454
457
|
logData(["Detecting lifecycle event", name], { options, debugMessages });
|
|
455
458
|
}
|
|
456
|
-
const shouldResolve =
|
|
457
|
-
(
|
|
459
|
+
const shouldResolve = frameId === state.topFrameId &&
|
|
460
|
+
(name === options.browserWaitUntil ||
|
|
461
|
+
(timeoutState.timeoutId && NETWORK_STATES.indexOf(name) < NETWORK_STATES.indexOf(options.browserWaitUntil)));
|
|
458
462
|
if (shouldResolve) {
|
|
463
|
+
// the delay is restarted when the page reaches a further state, so
|
|
464
|
+
// that it is captured once it stopped settling
|
|
459
465
|
clearTimeout(timeoutState.timeoutId);
|
|
460
466
|
logData([`Waiting ${options.browserWaitUntilDelay} ms`], { options, debugMessages });
|
|
461
|
-
setTimeout(() => {
|
|
467
|
+
timeoutState.timeoutId = setTimeout(() => {
|
|
462
468
|
logData(["Detecting page ready"], { options, debugMessages });
|
|
463
469
|
cleanup();
|
|
464
470
|
resolve();
|
|
@@ -467,49 +473,43 @@ function createLifecycleEventHandler(state, timeoutState, resolve, cleanup, { op
|
|
|
467
473
|
};
|
|
468
474
|
};
|
|
469
475
|
|
|
470
|
-
function createFrameNavigatedHandler(
|
|
476
|
+
function createFrameNavigatedHandler(timeoutState, reject, cleanup, { options, debugMessages }) {
|
|
471
477
|
const UNREACHABLE_URL_ERROR_MESSAGE = "Unreachable URL";
|
|
472
478
|
return ({ params }) => {
|
|
473
479
|
const { frame } = params;
|
|
474
|
-
if (!frame.parentId) {
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
} else {
|
|
480
|
-
logData(["Detecting top frame ID"], { options, debugMessages });
|
|
481
|
-
state.topFrameId = frame.id;
|
|
482
|
-
}
|
|
480
|
+
if (!frame.parentId && frame.unreachableUrl) {
|
|
481
|
+
logData(["Detecting unreachable URL", frame.unreachableUrl], { options, debugMessages });
|
|
482
|
+
clearTimeout(timeoutState.timeoutId);
|
|
483
|
+
cleanup();
|
|
484
|
+
reject(new Error(UNREACHABLE_URL_ERROR_MESSAGE + ": " + frame.unreachableUrl));
|
|
483
485
|
}
|
|
484
486
|
};
|
|
485
487
|
}
|
|
486
488
|
|
|
487
|
-
async function
|
|
488
|
-
const CONTEXT_NOT_FOUND_ERROR_MESSAGE = "Execution context not found for SingleFile world";
|
|
489
|
+
async function getSingleFileContext({ Page, Runtime }, topFrameId, { options, debugMessages }) {
|
|
489
490
|
const SINGLE_FILE_DETECTION_TEST = "typeof singlefile !== 'undefined'";
|
|
490
491
|
const NO_VALID_CONTEXT_ERROR_MESSAGE = "No valid SingleFile execution context found";
|
|
491
492
|
logData(["Getting execution context"], { options, debugMessages });
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
}
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
}
|
|
493
|
+
// the world already exists, so asking for it by name returns the context the
|
|
494
|
+
// injected script ran in instead of creating another one
|
|
495
|
+
const { executionContextId } = await Page.createIsolatedWorld({
|
|
496
|
+
frameId: topFrameId,
|
|
497
|
+
worldName: SINGLE_FILE_WORLD_NAME
|
|
498
|
+
});
|
|
499
|
+
// an empty world is returned when the script could not be injected, so the
|
|
500
|
+
// context is checked before it is used to capture the page
|
|
501
|
+
const { result } = await Runtime.evaluate({
|
|
502
|
+
expression: SINGLE_FILE_DETECTION_TEST,
|
|
503
|
+
contextId: executionContextId
|
|
504
|
+
});
|
|
505
|
+
if (result.value !== true) {
|
|
506
|
+
throw new Error(NO_VALID_CONTEXT_ERROR_MESSAGE);
|
|
507
507
|
}
|
|
508
|
-
|
|
508
|
+
return executionContextId;
|
|
509
509
|
}
|
|
510
510
|
|
|
511
|
-
function setupPageDataCapture({ Runtime },
|
|
512
|
-
return new Promise(resolve => {
|
|
511
|
+
function setupPageDataCapture({ Runtime }, _contextId, { options, debugMessages }) {
|
|
512
|
+
return new Promise((resolve, reject) => {
|
|
513
513
|
let pageDataResponse = "";
|
|
514
514
|
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ({ params }) => {
|
|
515
515
|
if (params.name === SET_PAGE_DATA_FUNCTION_NAME) {
|
|
@@ -518,11 +518,15 @@ function setupPageDataCapture({ Runtime }, contextId, { options, debugMessages }
|
|
|
518
518
|
pageDataResponse += payload;
|
|
519
519
|
} else {
|
|
520
520
|
logData(["Setting page data"], { options, debugMessages });
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
result.content
|
|
521
|
+
try {
|
|
522
|
+
const result = JSON.parse(pageDataResponse);
|
|
523
|
+
if (result.content instanceof Array) {
|
|
524
|
+
result.content = new Uint8Array(result.content);
|
|
525
|
+
}
|
|
526
|
+
resolve(result);
|
|
527
|
+
} catch (error) {
|
|
528
|
+
reject(error);
|
|
524
529
|
}
|
|
525
|
-
resolve(result);
|
|
526
530
|
}
|
|
527
531
|
}
|
|
528
532
|
});
|
|
@@ -534,20 +538,20 @@ async function setupBindings({ Page, Runtime }, contextId, { options, debugMessa
|
|
|
534
538
|
if (options.embedScreenshot && options.compressContent) {
|
|
535
539
|
await setupScreenshotCapture({ Page, Runtime }, contextId, { options, debugMessages });
|
|
536
540
|
}
|
|
537
|
-
if (options.embedPdf) {
|
|
541
|
+
if (options.embedPdf && options.compressContent) {
|
|
538
542
|
await setupPdfCapture({ Page, Runtime }, contextId, { options, debugMessages });
|
|
539
543
|
}
|
|
540
544
|
await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextId: contextId });
|
|
541
|
-
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, async ({ params }) => {
|
|
545
|
+
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
|
|
542
546
|
if (params.name === FETCH_FUNCTION_NAME) {
|
|
543
547
|
await handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages });
|
|
544
548
|
}
|
|
545
|
-
});
|
|
549
|
+
}, { options, debugMessages }));
|
|
546
550
|
}
|
|
547
551
|
|
|
548
552
|
async function setupScreenshotCapture({ Page, Runtime }, contextId, { options, debugMessages }) {
|
|
549
553
|
await Runtime.addBinding({ name: CAPTURE_SCREENSHOT_FUNCTION_NAME, executionContextId: contextId });
|
|
550
|
-
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, async ({ params }) => {
|
|
554
|
+
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
|
|
551
555
|
if (params.name === CAPTURE_SCREENSHOT_FUNCTION_NAME) {
|
|
552
556
|
logData(["Capturing screenshot"], { options, debugMessages });
|
|
553
557
|
try {
|
|
@@ -558,7 +562,7 @@ async function setupScreenshotCapture({ Page, Runtime }, contextId, { options, d
|
|
|
558
562
|
await callBrowserFunction({ Runtime }, contextId, SET_SCREENSHOT_FUNCTION_NAME, [""]);
|
|
559
563
|
}
|
|
560
564
|
}
|
|
561
|
-
});
|
|
565
|
+
}, { options, debugMessages }));
|
|
562
566
|
|
|
563
567
|
function parseScreenshotOptions(optionsString) {
|
|
564
568
|
const PNG_FORMAT = "png";
|
|
@@ -577,8 +581,8 @@ async function setupScreenshotCapture({ Page, Runtime }, contextId, { options, d
|
|
|
577
581
|
|
|
578
582
|
async function setupPdfCapture({ Page, Runtime }, contextId, { options, debugMessages }) {
|
|
579
583
|
await Runtime.addBinding({ name: PRINT_TO_PDF_FUNCTION_NAME, executionContextId: contextId });
|
|
580
|
-
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, async ({ params }) => {
|
|
581
|
-
if (params.name
|
|
584
|
+
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
|
|
585
|
+
if (params.name === PRINT_TO_PDF_FUNCTION_NAME) {
|
|
582
586
|
logData(["Printing to PDF", options.embedPdfOptions || ""], { options, debugMessages });
|
|
583
587
|
const pdfOptions = parsePdfOptions(options.embedPdfOptions);
|
|
584
588
|
try {
|
|
@@ -588,7 +592,7 @@ async function setupPdfCapture({ Page, Runtime }, contextId, { options, debugMes
|
|
|
588
592
|
await callBrowserFunction({ Runtime }, contextId, SET_PDF_FUNCTION_NAME, [""]);
|
|
589
593
|
}
|
|
590
594
|
}
|
|
591
|
-
});
|
|
595
|
+
}, { options, debugMessages }));
|
|
592
596
|
|
|
593
597
|
function parsePdfOptions(optionsString) {
|
|
594
598
|
let pdfOptions = {};
|
|
@@ -671,7 +675,10 @@ async function capturePageData({ Runtime }, contextId, { options, debugMessages
|
|
|
671
675
|
}
|
|
672
676
|
}
|
|
673
677
|
|
|
674
|
-
async function disableCdpDomains({ Console, Network, Page, Runtime }, { options }) {
|
|
678
|
+
async function disableCdpDomains({ Console, Fetch, Network, Page, Runtime }, { options }) {
|
|
679
|
+
// disabled first, so that the requests left paused are resumed by the
|
|
680
|
+
// browser instead of being held until the target is closed
|
|
681
|
+
await Fetch.disable();
|
|
675
682
|
await Runtime.disable();
|
|
676
683
|
await Page.disable();
|
|
677
684
|
if (options.httpHeaders) {
|
|
@@ -708,6 +715,18 @@ function attachDebugInfo(error, { options, consoleMessages, debugMessages }) {
|
|
|
708
715
|
}
|
|
709
716
|
}
|
|
710
717
|
|
|
718
|
+
function ignoringErrors(listener, pageContext) {
|
|
719
|
+
return async event => {
|
|
720
|
+
try {
|
|
721
|
+
await listener(event);
|
|
722
|
+
} catch (error) {
|
|
723
|
+
// the commands sent while the target is closing are rejected, and an
|
|
724
|
+
// error thrown here would be reported as an unhandled rejection
|
|
725
|
+
logData(["Ignoring event listener error", error.message], pageContext);
|
|
726
|
+
}
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
|
|
711
730
|
function logData(data, { options, debugMessages }) {
|
|
712
731
|
if (options.debugMessagesFile) {
|
|
713
732
|
debugMessages.push([Date.now(), data]);
|
package/lib/deno-polyfill.js
CHANGED
|
@@ -29,11 +29,6 @@ import * as path from "path";
|
|
|
29
29
|
|
|
30
30
|
const DENO_RUNTIME_DETECTED = typeof Deno !== "undefined";
|
|
31
31
|
|
|
32
|
-
const NPM_MODULES = {
|
|
33
|
-
"ws": "ws",
|
|
34
|
-
"simple-cdp": "simple-cdp",
|
|
35
|
-
};
|
|
36
|
-
|
|
37
32
|
const NODE_MODULES = {
|
|
38
33
|
"fs": "node:fs/promises",
|
|
39
34
|
"os": "node:os",
|
|
@@ -68,7 +63,9 @@ const Command = DENO_RUNTIME_DETECTED ? Deno.Command : class Command {
|
|
|
68
63
|
}
|
|
69
64
|
async spawn() {
|
|
70
65
|
const childProcess = await import(NODE_MODULES["child_process"]);
|
|
71
|
-
const
|
|
66
|
+
const stdio = [this.options.stdin, this.options.stdout, this.options.stderr]
|
|
67
|
+
.map(config => config == "null" ? "ignore" : config == "piped" ? "pipe" : "inherit");
|
|
68
|
+
const child = childProcess.spawn(this.path, this.options.args, { stdio });
|
|
72
69
|
|
|
73
70
|
await new Promise((resolve, reject) => {
|
|
74
71
|
child.on("spawn", () => resolve());
|
|
@@ -81,14 +78,8 @@ const Command = DENO_RUNTIME_DETECTED ? Deno.Command : class Command {
|
|
|
81
78
|
});
|
|
82
79
|
});
|
|
83
80
|
return {
|
|
84
|
-
status: new Promise(
|
|
85
|
-
child.on("exit", code => {
|
|
86
|
-
if (code === 0 || code === 143 || code === 130 || code === null) {
|
|
87
|
-
resolve();
|
|
88
|
-
} else {
|
|
89
|
-
reject(new Error(`Process exited with code ${code}`));
|
|
90
|
-
}
|
|
91
|
-
});
|
|
81
|
+
status: new Promise(resolve => {
|
|
82
|
+
child.on("exit", (code, signal) => resolve({ success: code === 0, code, signal }));
|
|
92
83
|
}),
|
|
93
84
|
kill() {
|
|
94
85
|
child.kill();
|
|
@@ -122,22 +113,21 @@ const DenoAPI = {
|
|
|
122
113
|
const pathAPI = {
|
|
123
114
|
dirname,
|
|
124
115
|
toFileUrl,
|
|
116
|
+
fromFileUrl,
|
|
125
117
|
SEPARATOR: DENO_RUNTIME_DETECTED ? path.SEPARATOR : path.sep
|
|
126
118
|
};
|
|
127
119
|
|
|
128
120
|
const isDeno = DENO_RUNTIME_DETECTED;
|
|
129
121
|
|
|
130
|
-
|
|
122
|
+
// the connection to the browser relies on the WebSocket of the runtime, which
|
|
123
|
+
// Node.js provides since 22.4.0 and can still be turned off with
|
|
124
|
+
// --no-experimental-websocket
|
|
125
|
+
if (typeof globalThis.WebSocket !== "function") {
|
|
126
|
+
throw new Error("WebSocket is not available, Node.js 22.4.0 or later is required");
|
|
127
|
+
}
|
|
131
128
|
|
|
132
129
|
export { DenoAPI as Deno, pathAPI as path, isDeno };
|
|
133
130
|
|
|
134
|
-
async function initGlobalThisProperties() {
|
|
135
|
-
if (!DENO_RUNTIME_DETECTED) {
|
|
136
|
-
const { WebSocket } = await import(getNPMModule("ws"));
|
|
137
|
-
globalThis.WebSocket = WebSocket;
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
|
|
141
131
|
async function readFile(path) {
|
|
142
132
|
if (DENO_RUNTIME_DETECTED) {
|
|
143
133
|
return Deno.readFile(path);
|
|
@@ -219,21 +209,21 @@ async function remove(path, options = {}) {
|
|
|
219
209
|
}
|
|
220
210
|
|
|
221
211
|
function exit(code) {
|
|
212
|
+
if (code == "SIGINT") {
|
|
213
|
+
code = 130;
|
|
214
|
+
} else if (code == "SIGTERM") {
|
|
215
|
+
code = 143;
|
|
216
|
+
}
|
|
222
217
|
if (DENO_RUNTIME_DETECTED) {
|
|
223
218
|
Deno.exit(code);
|
|
224
219
|
} else {
|
|
225
|
-
if (code == "SIGINT") {
|
|
226
|
-
code = 130;
|
|
227
|
-
} else if (code == "SIGTERM") {
|
|
228
|
-
code = 143;
|
|
229
|
-
}
|
|
230
220
|
process.exit(code);
|
|
231
221
|
}
|
|
232
222
|
}
|
|
233
223
|
|
|
234
224
|
function addSignalListener(signal, listener) {
|
|
235
225
|
if (DENO_RUNTIME_DETECTED) {
|
|
236
|
-
Deno.addSignalListener(signal, listener);
|
|
226
|
+
Deno.addSignalListener(signal, () => listener(signal));
|
|
237
227
|
} else {
|
|
238
228
|
process.once(signal, listener);
|
|
239
229
|
}
|
|
@@ -261,6 +251,11 @@ async function toFileUrl(filePath) {
|
|
|
261
251
|
}
|
|
262
252
|
}
|
|
263
253
|
|
|
264
|
-
function
|
|
265
|
-
|
|
254
|
+
async function fromFileUrl(fileUrl) {
|
|
255
|
+
if (DENO_RUNTIME_DETECTED) {
|
|
256
|
+
return path.fromFileUrl(fileUrl);
|
|
257
|
+
} else {
|
|
258
|
+
const url = await import(NODE_MODULES["url"]);
|
|
259
|
+
return url.fileURLToPath(fileUrl);
|
|
260
|
+
}
|
|
266
261
|
}
|