polite-crawler 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +557 -10
  2. package/build/browser-CKRNbIDT.mjs +62 -0
  3. package/build/browser-CKRNbIDT.mjs.map +1 -0
  4. package/build/cli.mjs +795 -121
  5. package/build/cli.mjs.map +1 -1
  6. package/build/diff-Dk2eSvi-.mjs +648 -0
  7. package/build/diff-Dk2eSvi-.mjs.map +1 -0
  8. package/build/duration-iIuMMofi.mjs +38 -0
  9. package/build/duration-iIuMMofi.mjs.map +1 -0
  10. package/build/extract-DcwNrTK7.mjs +1793 -0
  11. package/build/extract-DcwNrTK7.mjs.map +1 -0
  12. package/build/{html-nWYFVsRU.d.mts → html-BACljlkf.d.mts} +8 -9
  13. package/build/{html-oNpGAHwi.mjs → html-DkQUAE_d.mjs} +18 -39
  14. package/build/html-DkQUAE_d.mjs.map +1 -0
  15. package/build/html.d.mts +2 -1
  16. package/build/html.mjs +3 -1
  17. package/build/index.d.mts +94 -4
  18. package/build/index.mjs +13 -5
  19. package/build/log-Dx-wXjJl.mjs +342 -0
  20. package/build/log-Dx-wXjJl.mjs.map +1 -0
  21. package/build/mcp.d.mts +7 -4
  22. package/build/mcp.mjs +556 -32
  23. package/build/mcp.mjs.map +1 -1
  24. package/build/open-Ca1lcWR3.d.mts +2875 -0
  25. package/build/open-DnrWgkC0.mjs +3161 -0
  26. package/build/open-DnrWgkC0.mjs.map +1 -0
  27. package/build/{browser-gJ_h4AhB.mjs → render-C1CUraWF.mjs} +31 -74
  28. package/build/render-C1CUraWF.mjs.map +1 -0
  29. package/build/rolldown-runtime-D7D4PA-g.mjs +13 -0
  30. package/build/{open-DYZ7KL2A.mjs → runtime-k8mo1_7U.mjs} +986 -1657
  31. package/build/runtime-k8mo1_7U.mjs.map +1 -0
  32. package/build/urls-BHFeSyEK.mjs +62 -0
  33. package/build/urls-BHFeSyEK.mjs.map +1 -0
  34. package/build/urls-BTDSgDLC.d.mts +16 -0
  35. package/build/wacz-sfOLT8e9.mjs +275 -0
  36. package/build/wacz-sfOLT8e9.mjs.map +1 -0
  37. package/build/warc-VIPscWxY.mjs +728 -0
  38. package/build/warc-VIPscWxY.mjs.map +1 -0
  39. package/build/webbotauth-CjJdTAfE.mjs +568 -0
  40. package/build/webbotauth-CjJdTAfE.mjs.map +1 -0
  41. package/package.json +1 -1
  42. package/record.schema.json +15 -1
  43. package/build/browser-gJ_h4AhB.mjs.map +0 -1
  44. package/build/html-oNpGAHwi.mjs.map +0 -1
  45. package/build/open-DYZ7KL2A.mjs.map +0 -1
  46. package/build/open-eHcriTds.d.mts +0 -1499
  47. package/build/version-no12qgos.mjs +0 -11
  48. package/build/version-no12qgos.mjs.map +0 -1
@@ -0,0 +1,62 @@
1
+ import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
2
+ import { n as RenderRefused, t as NO_PLAYWRIGHT } from "./render-C1CUraWF.mjs";
3
+ import { basename, dirname, join } from "node:path";
4
+ import { existsSync } from "node:fs";
5
+ //#region src/browser.ts
6
+ var browser_exports = /* @__PURE__ */ __exportAll({
7
+ NO_BROWSER: () => NO_BROWSER,
8
+ browserMissing: () => browserMissing,
9
+ checkedRenderer: () => checkedRenderer
10
+ });
11
+ /** The text of the note when `playwright-core` is installed, but its browser is not. */
12
+ const NO_BROWSER = "rendering needs a browser: npx playwright-core install chromium-headless-shell";
13
+ /**
14
+ * This function tells if the default renderer can start a browser. It gives no value when it can,
15
+ * and the reason when it cannot. The command line and the MCP server use it one time at start.
16
+ * Then a missing browser costs no wait and no error capture for each page that needs JavaScript.
17
+ *
18
+ * The check does not start the browser. It finds the full Chromium build or the headless shell of
19
+ * the same revision on the disk. When the check cannot find the answer, it gives no value, and the
20
+ * renderer then tries.
21
+ */
22
+ const browserMissing = async (load = () => import("playwright-core")) => {
23
+ let chromium;
24
+ try {
25
+ ({chromium} = await load());
26
+ } catch {
27
+ return NO_PLAYWRIGHT;
28
+ }
29
+ try {
30
+ const executable = chromium.executablePath();
31
+ if (existsSync(executable)) return void 0;
32
+ let folder = dirname(executable);
33
+ while (dirname(folder) !== folder && !/^chromium-\d+$/.test(basename(folder))) folder = dirname(folder);
34
+ const revision = /^chromium-(\d+)$/.exec(basename(folder))?.[1];
35
+ if (revision === void 0) return void 0;
36
+ return existsSync(join(dirname(folder), `chromium_headless_shell-${revision}`)) ? void 0 : NO_BROWSER;
37
+ } catch {
38
+ return;
39
+ }
40
+ };
41
+ /**
42
+ * This function gives a renderer that checks one time, before its first page, that the browser is
43
+ * on the disk (`browserMissing`). With no browser, each page gets a `RenderRefused` with the reason
44
+ * `browser` at once, and the renderer opens nothing. `createRuntime` puts its default renderer in
45
+ * it, so a library with no browser costs no wait and no error capture.
46
+ */
47
+ const checkedRenderer = (renderer, check = () => browserMissing()) => {
48
+ let missing;
49
+ return {
50
+ async render(url, options) {
51
+ missing ??= check().catch(() => void 0);
52
+ const reason = await missing;
53
+ if (reason !== void 0) throw new RenderRefused("browser", reason);
54
+ return renderer.render(url, options);
55
+ },
56
+ close: () => renderer.close()
57
+ };
58
+ };
59
+ //#endregion
60
+ export { browser_exports as n, checkedRenderer as r, browserMissing as t };
61
+
62
+ //# sourceMappingURL=browser-CKRNbIDT.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"browser-CKRNbIDT.mjs","names":[],"sources":["../src/browser.ts"],"sourcesContent":["import { existsSync } from \"node:fs\";\nimport { basename, dirname, join } from \"node:path\";\nimport type { Renderer } from \"./render.ts\";\nimport { NO_PLAYWRIGHT, RenderRefused } from \"./render.ts\";\n\n/** The text of the note when `playwright-core` is installed, but its browser is not. */\nexport const NO_BROWSER =\n \"rendering needs a browser: npx playwright-core install chromium-headless-shell\";\n\n/** This is the part of `playwright-core` that the check uses. */\ninterface Chromium {\n executablePath(): string;\n}\n\n/**\n * This function tells if the default renderer can start a browser. It gives no value when it can,\n * and the reason when it cannot. The command line and the MCP server use it one time at start.\n * Then a missing browser costs no wait and no error capture for each page that needs JavaScript.\n *\n * The check does not start the browser. It finds the full Chromium build or the headless shell of\n * the same revision on the disk. When the check cannot find the answer, it gives no value, and the\n * renderer then tries.\n */\nexport const browserMissing = async (\n load: () => Promise<{ readonly chromium: Chromium }> = () => import(\"playwright-core\"),\n): Promise<string | undefined> => {\n let chromium: Chromium;\n try {\n ({ chromium } = await load());\n } catch {\n return NO_PLAYWRIGHT;\n }\n try {\n const executable = chromium.executablePath();\n if (existsSync(executable)) return undefined;\n // the headless shell is in a folder next to the folder of the full build, with the same revision\n let folder = dirname(executable);\n while (dirname(folder) !== folder && !/^chromium-\\d+$/.test(basename(folder)))\n folder = dirname(folder);\n const revision = /^chromium-(\\d+)$/.exec(basename(folder))?.[1];\n if (revision === undefined) return undefined;\n return existsSync(join(dirname(folder), `chromium_headless_shell-${revision}`))\n ? undefined\n : NO_BROWSER;\n } catch {\n return undefined;\n }\n};\n\n/**\n * This function gives a renderer that checks one time, before its first page, that the browser is\n * on the disk (`browserMissing`). With no browser, each page gets a `RenderRefused` with the reason\n * `browser` at once, and the renderer opens nothing. `createRuntime` puts its default renderer in\n * it, so a library with no browser costs no wait and no error capture.\n */\nexport const checkedRenderer = (\n renderer: Renderer,\n check: () => Promise<string | undefined> = () => browserMissing(),\n): Renderer => {\n let missing: Promise<string | undefined> | undefined;\n return {\n async render(url, options) {\n missing ??= check().catch(() => undefined);\n const reason = await missing;\n if (reason !== undefined) throw new RenderRefused(\"browser\", reason);\n return renderer.render(url, options);\n },\n close: () => renderer.close(),\n };\n};\n"],"mappings":";;;;;;;;;;;AAMA,MAAa,aACX;;;;;;;;;;AAgBF,MAAa,iBAAiB,OAC5B,aAA6D,OAAO,uBACpC;CAChC,IAAI;CACJ,IAAI;EACF,CAAC,CAAE,YAAa,MAAM,KAAK;CAC7B,QAAQ;EACN,OAAO;CACT;CACA,IAAI;EACF,MAAM,aAAa,SAAS,eAAe;EAC3C,IAAI,WAAW,UAAU,GAAG,OAAO,KAAA;EAEnC,IAAI,SAAS,QAAQ,UAAU;EAC/B,OAAO,QAAQ,MAAM,MAAM,UAAU,CAAC,iBAAiB,KAAK,SAAS,MAAM,CAAC,GAC1E,SAAS,QAAQ,MAAM;EACzB,MAAM,WAAW,mBAAmB,KAAK,SAAS,MAAM,CAAC,CAAC,GAAG;EAC7D,IAAI,aAAa,KAAA,GAAW,OAAO,KAAA;EACnC,OAAO,WAAW,KAAK,QAAQ,MAAM,GAAG,2BAA2B,UAAU,CAAC,IAC1E,KAAA,IACA;CACN,QAAQ;EACN;CACF;AACF;;;;;;;AAQA,MAAa,mBACX,UACA,cAAiD,eAAe,MACnD;CACb,IAAI;CACJ,OAAO;EACL,MAAM,OAAO,KAAK,SAAS;GACzB,YAAY,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;GACzC,MAAM,SAAS,MAAM;GACrB,IAAI,WAAW,KAAA,GAAW,MAAM,IAAI,cAAc,WAAW,MAAM;GACnE,OAAO,SAAS,OAAO,KAAK,OAAO;EACrC;EACA,aAAa,SAAS,MAAM;CAC9B;AACF"}