webforai 1.5.1 → 1.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/dist/bin.js +258 -278
  2. package/dist/cjs/extract-hast/index.js +4 -3
  3. package/dist/cjs/extract-hast/readability.js +28 -61
  4. package/dist/cjs/extract-hast/strongly.js +2 -1
  5. package/dist/cjs/extract-hast/utils.js +0 -15
  6. package/dist/cjs/html-to-mdast.js +6 -6
  7. package/dist/cjs/link-replacer.js +10 -5
  8. package/dist/cjs/loaders/playwright.js +36 -7
  9. package/dist/cjs/{cli/constants.js → mdast-handlers/custom-br-handler.js} +9 -12
  10. package/dist/cjs/mdast-to-markdown.js +1 -1
  11. package/dist/cjs/utils/hast-utils.js +71 -0
  12. package/dist/extract-hast/index.js +4 -3
  13. package/dist/extract-hast/readability.js +29 -62
  14. package/dist/extract-hast/strongly.js +2 -1
  15. package/dist/extract-hast/utils.js +0 -14
  16. package/dist/html-to-mdast.js +6 -6
  17. package/dist/link-replacer.js +10 -5
  18. package/dist/loaders/playwright.js +36 -7
  19. package/dist/mdast-handlers/custom-br-handler.js +8 -0
  20. package/dist/mdast-to-markdown.js +1 -1
  21. package/dist/types/cli/bin.js.map +1 -1
  22. package/dist/types/cli/commands/webforai/index.d.ts +6 -0
  23. package/dist/types/cli/commands/webforai/index.js.map +1 -0
  24. package/dist/types/cli/commands/webforai/loadHtml.d.ts +3 -0
  25. package/dist/types/cli/commands/webforai/loadHtml.js.map +1 -0
  26. package/dist/types/extract-hast/index.d.ts +9 -4
  27. package/dist/types/extract-hast/index.js.map +1 -1
  28. package/dist/types/extract-hast/readability.d.ts +2 -1
  29. package/dist/types/extract-hast/readability.js.map +1 -1
  30. package/dist/types/extract-hast/strongly.d.ts +2 -1
  31. package/dist/types/extract-hast/strongly.js.map +1 -1
  32. package/dist/types/extract-hast/utils.d.ts +0 -1
  33. package/dist/types/extract-hast/utils.js.map +1 -1
  34. package/dist/types/html-to-mdast.d.ts +4 -0
  35. package/dist/types/html-to-mdast.js.map +1 -1
  36. package/dist/types/link-replacer.d.ts +1 -1
  37. package/dist/types/link-replacer.js.map +1 -1
  38. package/dist/types/loaders/playwright.d.ts +8 -2
  39. package/dist/types/loaders/playwright.js.map +1 -1
  40. package/dist/types/mdast-handlers/custom-br-handler.d.ts +2 -0
  41. package/dist/types/mdast-handlers/custom-br-handler.js.map +1 -0
  42. package/dist/types/mdast-to-markdown.js.map +1 -1
  43. package/dist/types/utils/hast-utils.d.ts +4 -0
  44. package/dist/types/utils/hast-utils.js.map +1 -0
  45. package/dist/utils/hast-utils.js +46 -0
  46. package/package.json +22 -10
  47. package/dist/cjs/cli/bin.js +0 -34
  48. package/dist/cjs/cli/commands/handlers/index.js +0 -84
  49. package/dist/cjs/cli/commands/handlers/loadHtml.js +0 -127
  50. package/dist/cjs/cli/helpers/assertContinue.js +0 -34
  51. package/dist/cjs/cli/helpers/inputOutputPath.js +0 -89
  52. package/dist/cjs/cli/helpers/inputSourcePath.js +0 -64
  53. package/dist/cjs/cli/helpers/selectExtractMode.js +0 -39
  54. package/dist/cjs/cli/helpers/selectLoader.js +0 -44
  55. package/dist/cjs/cli/utils.js +0 -122
  56. package/dist/cjs/html-to-markdown.test.js +0 -118
  57. package/dist/cjs/link-replacer.test.js +0 -47
  58. package/dist/cjs/loaders/fetch.test.js +0 -9
  59. package/dist/cjs/loaders/playwright.test.js +0 -17
  60. package/dist/cjs/loaders/puppeteer.test.js +0 -17
  61. package/dist/cli/bin.js +0 -12
  62. package/dist/cli/commands/handlers/index.js +0 -51
  63. package/dist/cli/commands/handlers/loadHtml.js +0 -94
  64. package/dist/cli/constants.js +0 -9
  65. package/dist/cli/helpers/assertContinue.js +0 -11
  66. package/dist/cli/helpers/inputOutputPath.js +0 -56
  67. package/dist/cli/helpers/inputSourcePath.js +0 -31
  68. package/dist/cli/helpers/selectExtractMode.js +0 -16
  69. package/dist/cli/helpers/selectLoader.js +0 -21
  70. package/dist/cli/utils.js +0 -85
  71. package/dist/html-to-markdown.test.js +0 -118
  72. package/dist/link-replacer.test.js +0 -47
  73. package/dist/loaders/fetch.test.js +0 -9
  74. package/dist/loaders/playwright.test.js +0 -17
  75. package/dist/loaders/puppeteer.test.js +0 -17
  76. package/dist/types/cli/commands/handlers/index.d.ts +0 -13
  77. package/dist/types/cli/commands/handlers/index.js.map +0 -1
  78. package/dist/types/cli/commands/handlers/loadHtml.d.ts +0 -2
  79. package/dist/types/cli/commands/handlers/loadHtml.js.map +0 -1
@@ -1,17 +0,0 @@
1
- // src/loaders/playwright.test.ts
2
- import { chromium } from "playwright";
3
- import { describe, expect, it } from "vitest";
4
- import { loadHtml } from "./playwright.js";
5
- describe("Playwright loader", () => {
6
- it("should load the HTML of a URL", async () => {
7
- const html = await loadHtml("https://example.com");
8
- expect(html).toContain("Example Domain");
9
- });
10
- it("should load the HTML of a URL using a custom context", async () => {
11
- const context = await chromium.launch({ headless: true });
12
- const html = await loadHtml("https://example.com", context);
13
- expect(html).toContain("Example Domain");
14
- expect(context.isConnected()).toBe(true);
15
- await context.close();
16
- });
17
- });
@@ -1,17 +0,0 @@
1
- // src/loaders/puppeteer.test.ts
2
- import { chromium } from "playwright.js";
3
- import { describe, expect, it } from "vitest";
4
- import { loadHtml } from "./puppeteer.js";
5
- describe("Puppeteer loader", () => {
6
- it("should load the HTML of a URL", async () => {
7
- const html = await loadHtml("https://example.com");
8
- expect(html).toContain("Example Domain");
9
- });
10
- it("should load the HTML of a URL using a custom puppeteer context", async () => {
11
- const context = await chromium.launch({ headless: true });
12
- const html = await loadHtml("https://example.com", { headless: true });
13
- expect(html).toContain("Example Domain");
14
- expect(context.isConnected()).toBe(true);
15
- await context.close();
16
- });
17
- });
@@ -1,13 +0,0 @@
1
- export type LoaderOptions = {
2
- type: "web";
3
- loader: "playwright" | "pw" | "fetch" | string;
4
- } | {
5
- type: "local";
6
- loader: null;
7
- };
8
- export declare const webforaiCommand: (initialPath: string, initialOutputPath: string, options: {
9
- mode?: string;
10
- loader: string;
11
- debug?: boolean;
12
- baseUrl?: string;
13
- }) => Promise<void>;
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/cli/commands/handlers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,MAAM,aAAa,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AAC/E,MAAM,sBAAsB,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAM3F,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EACnC,WAAmB,EACnB,iBAAyB,EACzB,OAA6E,EAC5E,EAAE;IACH,MAAM,UAAU,GAAG,WAAW,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;IAC5D,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,MAAM,GAAkB,KAAK,CAAC,UAAU,CAAC;QAC9C,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,YAAY,EAAE,CAAC,EAAE;QACnE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,WAAW,MAAM,EAAE,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,UAAU,GAAG,iBAAiB,IAAI,CAAC,MAAM,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;IAC5E,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC;IACzD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACnD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,EAAE;QACxC,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO;QACzD,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,sBAAsB,CAAC;KAC3D,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAE3C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IACD,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAEvC,KAAK,CAAC,kCAAkC,CAAC,CAAC;AAC3C,CAAC,CAAC"}
@@ -1,2 +0,0 @@
1
- import type { LoaderOptions } from "./index";
2
- export declare const loadHtml: (sourcePath: string, loader: LoaderOptions) => Promise<string>;
@@ -1 +0,0 @@
1
- {"version":3,"file":"loadHtml.js","sourceRoot":"","sources":["../../../../../src/cli/commands/handlers/loadHtml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EAAE,MAAM,EAAE,OAAO,IAAI,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACnF,OAAO,EAAE,MAAM,YAAY,CAAC;AAE5B,OAAO,EAAE,CAAC,EAAE,MAAM,IAAI,CAAC;AAKvB,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,UAAkB,EAAE,MAAqB,EAAmB,EAAE;IAC5F,MAAM,WAAW,GAAG,YAAY,EAAE,CAAC;IAEnC,IAAI,CAAC;QACJ,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,OAAO;gBACX,OAAO,MAAM,aAAa,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;YACrD,KAAK,KAAK;gBACT,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;oBACvB,KAAK,OAAO;wBACX,OAAO,MAAM,aAAa,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;oBACrD,KAAK,YAAY;wBAChB,OAAO,MAAM,kBAAkB,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;oBAC1D;wBACC,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;gBACpC,CAAC;YACF;gBACC,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,CAAC;QACpD,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,KAAK,EAAE,UAAkB,EAAE,WAAwB,EAAmB,EAAE;IAC7F,WAAW,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,IAAI,CAAC;QACJ,MAAM,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACvD,WAAW,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IACnD,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,KAAK,EAAE,UAAkB,EAAE,WAAwB,EAAmB,EAAE;IAC7F,WAAW,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC3C,WAAW,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,KAAK,EAAE,UAAkB,EAAE,WAAwB,EAAmB,EAAE;IAClG,MAAM,aAAa,GAAG,YAAY,EAAE,CAAC;IACrC,aAAa,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IAE7C,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAC9C,IAAI,OAA0B,CAAC;IAE/B,IAAI,CAAC;QACJ,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC7C,aAAa,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,2BAA2B,CAAC,KAAc,CAAC,CAAC;QAClD,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;IAC9C,CAAC;IAED,WAAW,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACtC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAElD,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IAEtB,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,2BAA2B,GAAG,KAAK,EAAE,KAAY,EAAiB,EAAE;IACzE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;QACvD,MAAM,KAAK,CAAC;IACb,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC;QAC/B,OAAO,EAAE,4EAA4E;KACrF,CAAC,CAAC;IACH,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAC1B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,kBAAkB,GAAG,OAAO,EAAE,CAAC;IACrC,kBAAkB,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAE7D,IAAI,CAAC;QACJ,MAAM,CAAC,CAAA,wBAAwB,CAAC;QAChC,kBAAkB,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAC5D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC,CAAC;QAC3E,MAAM,KAAK,CAAC;IACb,CAAC;AACF,CAAC,CAAC"}