single-file-cli 2.2.2 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. package/Dockerfile +1 -0
  2. package/README.MD +6 -2
  3. package/build.sh +1 -1
  4. package/deno.json +1 -1
  5. package/lib/archive-packager.js +3 -0
  6. package/lib/cdp-client-util.js +4 -1
  7. package/lib/cdp-client.js +24 -20
  8. package/lib/single-file-archive.js +6 -6
  9. package/lib/single-file-bundle.js +1 -1
  10. package/lib/version.js +1 -1
  11. package/options.js +8 -3
  12. package/package.json +1 -1
  13. package/single-file-cli-api.js +11 -2
  14. package/single-file-launcher.js +4 -1
  15. package/test/e2e/errors-file.test.js +11 -5
  16. package/test/e2e/urls-file.test.js +31 -0
  17. package/test/unit/archive-packager.test.js +15 -2
  18. package/test/unit/options.test.js +3 -0
package/Dockerfile CHANGED
@@ -14,5 +14,6 @@ ENTRYPOINT [ \
14
14
  "npx", \
15
15
  "single-file", \
16
16
  "--browser-executable-path", "/usr/bin/chromium-browser", \
17
+ "--browser-single-process=false", \
17
18
  "--output-directory", "./out/", \
18
19
  "--dump-content" ]
package/README.MD CHANGED
@@ -143,12 +143,16 @@ Make sure Chrome or a Chromium-based browser is installed in the default folder.
143
143
  single-file https://www.wikipedia.org --crawl-links=true --crawl-inner-links-only=false --crawl-external-links-max-depth=1 --crawl-rewrite-rule="^.*wikipedia.*$"
144
144
  ```
145
145
 
146
- - Save https://www.wikipedia.org and its internal links into a single self-extracting ZIP file
146
+ - Save http://info.cern.ch (i.e. the first website) and its internal links into a single self-extracting ZIP file, and mark the links to pages missing from the archive
147
147
 
148
148
  ```sh
149
- single-file https://www.wikipedia.org wikipedia.html --crawl-links=true --crawl-save-archive=true --compress-content=true
149
+ single-file http://info.cern.ch first-website.html --crawl-links=true --crawl-max-depth=2 --crawl-save-archive=true --crawl-save-archive-mark-unarchived-links=true --compress-content=true
150
150
  ```
151
151
 
152
+ - Exit code
153
+
154
+ The exit code is 0 when all the pages have been saved, 1 when at least one capture failed (the other pages of the batch or the crawl are still saved), and 255 when an error prevented the process from running.
155
+
152
156
  ## Compile executables
153
157
 
154
158
  - Compile executables into `/dist`
package/build.sh CHANGED
@@ -3,7 +3,7 @@
3
3
  mv package.json package.json.tmp
4
4
  mv deno.json deno.json.tmp
5
5
  mv deno.lock deno.lock.tmp
6
- deno install --vendor --quiet --minimum-dependency-age=0 "npm:single-file-core@1.5.92"
6
+ deno install --vendor --quiet --minimum-dependency-age=0 "npm:single-file-core@1.5.95"
7
7
  mv package.json.tmp package.json
8
8
  mv deno.json.tmp deno.json
9
9
  mv deno.lock.tmp deno.lock
package/deno.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@single-file/single-file-cli",
3
- "version": "2.2.2",
3
+ "version": "2.3.0",
4
4
  "description": "SingleFile CLI",
5
5
  "exports": {
6
6
  ".": "./single-file-cli-api.js"
package/lib/archive-packager.js CHANGED
@@ -59,6 +59,9 @@ async function createPagesArchive(pages, options) {
59
59
  if (options.markUnarchivedLinks) {
60
60
  manifest.markUnarchivedLinks = true;
61
61
  }
62
+ if (options.pageTransitions && options.pageTransitions != "auto") {
63
+ manifest.pageTransitions = options.pageTransitions;
64
+ }
62
65
  const pageData = {
63
66
  doctype: "<!DOCTYPE html>",
64
67
  content: "",
package/lib/cdp-client-util.js CHANGED
@@ -74,10 +74,13 @@ function waitForTimeout(abortSignal, maxDelay, errorMessage, errorCode) {
74
74
  reject(error);
75
75
  }, maxDelay);
76
76
 
77
+ // the promise must stay pending on abort: the abort fires after the
78
+ // Promise.race around this timeout has settled, and settling here would
79
+ // either make the race winner destructure undefined or raise an
80
+ // unhandled rejection
77
81
  function onAbort() {
78
82
  abortSignal.removeEventListener(ABORT_EVENT, onAbort);
79
83
  clearTimeout(timeoutId);
80
- resolve();
81
84
  }
82
85
  });
83
86
  }
package/lib/cdp-client.js CHANGED
@@ -115,8 +115,8 @@ async function getPageData(options) {
115
115
  await setupNetworkInterception(cdp, pageContext);
116
116
  await setupScriptInjection(cdp, pageContext);
117
117
  const contextId = await getContextId(cdp, pageContext);
118
- const pageDataPromise = setupPageDataCapture(cdp, contextId, pageContext);
119
- await setupBindings(cdp, contextId, pageContext);
118
+ const pageDataPromise = setupPageDataCapture(cdp, pageContext);
119
+ await setupBindings(cdp, pageContext);
120
120
  await capturePageData(cdp, contextId, pageContext);
121
121
  await disableCdpDomains(cdp, pageContext);
122
122
  return await finalizePageData(pageDataPromise, pageContext);
@@ -440,6 +440,9 @@ async function loadPage({ Page, Runtime }, { options, debugMessages }) {
440
440
  const LOAD_TIMEOUT_ERROR_MESSAGE = "Load timeout";
441
441
  await Runtime.enable();
442
442
  await Page.enable();
443
+ if (options.browserBypassCSP) {
444
+ await Page.setBypassCSP({ enabled: true });
445
+ }
443
446
  await Page.setLifecycleEventsEnabled({ enabled: true });
444
447
  // the ID of the top frame is stable, and it is read before the navigation is
445
448
  // triggered so that no event is missed while the browser answers
@@ -535,7 +538,8 @@ async function getSingleFileContext({ Page, Runtime }, topFrameId, { options, de
535
538
  // injected script ran in instead of creating another one
536
539
  const { executionContextId } = await Page.createIsolatedWorld({
537
540
  frameId: topFrameId,
538
- worldName: SINGLE_FILE_WORLD_NAME
541
+ worldName: SINGLE_FILE_WORLD_NAME,
542
+ contentSecurityPolicy: ""
539
543
  });
540
544
  // an empty world is returned when the script could not be injected, so the
541
545
  // context is checked before it is used to capture the page
@@ -549,7 +553,7 @@ async function getSingleFileContext({ Page, Runtime }, topFrameId, { options, de
549
553
  return executionContextId;
550
554
  }
551
555
 
552
- function setupPageDataCapture({ Runtime }, _contextId, { options, debugMessages }) {
556
+ function setupPageDataCapture({ Runtime }, { options, debugMessages }) {
553
557
  return new Promise((resolve, reject) => {
554
558
  let pageDataResponse = "";
555
559
  Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ({ params }) => {
@@ -574,33 +578,33 @@ function setupPageDataCapture({ Runtime }, _contextId, { options, debugMessages
574
578
  });
575
579
  }
576
580
 
577
- async function setupBindings({ Page, Runtime }, contextId, { options, debugMessages, fetchAbortController }) {
578
- await Runtime.addBinding({ name: SET_PAGE_DATA_FUNCTION_NAME, executionContextId: contextId });
581
+ async function setupBindings({ Page, Runtime }, { options, debugMessages, fetchAbortController }) {
582
+ await Runtime.addBinding({ name: SET_PAGE_DATA_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
579
583
  if (options.embedScreenshot && options.compressContent) {
580
- await setupScreenshotCapture({ Page, Runtime }, contextId, { options, debugMessages });
584
+ await setupScreenshotCapture({ Page, Runtime }, { options, debugMessages });
581
585
  }
582
586
  if (options.embedPdf && options.compressContent) {
583
- await setupPdfCapture({ Page, Runtime }, contextId, { options, debugMessages });
587
+ await setupPdfCapture({ Page, Runtime }, { options, debugMessages });
584
588
  }
585
- await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextId: contextId });
589
+ await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
586
590
  Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
587
591
  if (params.name === FETCH_FUNCTION_NAME) {
588
- await handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages, fetchAbortController });
592
+ await handleFetchRequest({ Runtime }, params, { options, debugMessages, fetchAbortController });
589
593
  }
590
594
  }, { options, debugMessages }));
591
595
  }
592
596
 
593
- async function setupScreenshotCapture({ Page, Runtime }, contextId, { options, debugMessages }) {
594
- await Runtime.addBinding({ name: CAPTURE_SCREENSHOT_FUNCTION_NAME, executionContextId: contextId });
597
+ async function setupScreenshotCapture({ Page, Runtime }, { options, debugMessages }) {
598
+ await Runtime.addBinding({ name: CAPTURE_SCREENSHOT_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
595
599
  Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
596
600
  if (params.name === CAPTURE_SCREENSHOT_FUNCTION_NAME) {
597
601
  logData(["Capturing screenshot"], { options, debugMessages });
598
602
  try {
599
603
  const screenshotOptions = parseScreenshotOptions(options.embedScreenshotOptions);
600
604
  const { data } = await Page.captureScreenshot(screenshotOptions);
601
- await callBrowserFunction({ Runtime }, contextId, SET_SCREENSHOT_FUNCTION_NAME, [data]);
605
+ await callBrowserFunction({ Runtime }, params.executionContextId, SET_SCREENSHOT_FUNCTION_NAME, [data]);
602
606
  } catch {
603
- await callBrowserFunction({ Runtime }, contextId, SET_SCREENSHOT_FUNCTION_NAME, [""]);
607
+ await callBrowserFunction({ Runtime }, params.executionContextId, SET_SCREENSHOT_FUNCTION_NAME, [""]);
604
608
  }
605
609
  }
606
610
  }, { options, debugMessages }));
@@ -620,17 +624,17 @@ async function setupScreenshotCapture({ Page, Runtime }, contextId, { options, d
620
624
  }
621
625
  }
622
626
 
623
- async function setupPdfCapture({ Page, Runtime }, contextId, { options, debugMessages }) {
624
- await Runtime.addBinding({ name: PRINT_TO_PDF_FUNCTION_NAME, executionContextId: contextId });
627
+ async function setupPdfCapture({ Page, Runtime }, { options, debugMessages }) {
628
+ await Runtime.addBinding({ name: PRINT_TO_PDF_FUNCTION_NAME, executionContextName: SINGLE_FILE_WORLD_NAME });
625
629
  Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
626
630
  if (params.name === PRINT_TO_PDF_FUNCTION_NAME) {
627
631
  logData(["Printing to PDF", options.embedPdfOptions || ""], { options, debugMessages });
628
632
  const pdfOptions = parsePdfOptions(options.embedPdfOptions);
629
633
  try {
630
634
  const { data } = await Page.printToPDF(pdfOptions);
631
- await callBrowserFunction({ Runtime }, contextId, SET_PDF_FUNCTION_NAME, [data]);
635
+ await callBrowserFunction({ Runtime }, params.executionContextId, SET_PDF_FUNCTION_NAME, [data]);
632
636
  } catch {
633
- await callBrowserFunction({ Runtime }, contextId, SET_PDF_FUNCTION_NAME, [""]);
637
+ await callBrowserFunction({ Runtime }, params.executionContextId, SET_PDF_FUNCTION_NAME, [""]);
634
638
  }
635
639
  }
636
640
  }, { options, debugMessages }));
@@ -648,8 +652,8 @@ async function setupPdfCapture({ Page, Runtime }, contextId, { options, debugMes
648
652
  }
649
653
  }
650
654
 
651
- async function handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages, fetchAbortController }) {
652
- const { payload } = params;
655
+ async function handleFetchRequest({ Runtime }, params, { options, debugMessages, fetchAbortController }) {
656
+ const { executionContextId: contextId, payload } = params;
653
657
  const { requestId, url, options: fetchOptions } = JSON.parse(payload);
654
658
  logData(["Fetching URL", url], { options, debugMessages });
655
659
  try {