webforai 1.5.1 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/dist/bin.js +277 -334
  2. package/dist/cjs/cli/bin.js +3 -4
  3. package/dist/cjs/cli/commands/{handlers → webforai}/index.js +33 -31
  4. package/dist/cjs/cli/commands/webforai/loadHtml.js +93 -0
  5. package/dist/cjs/cli/utils.js +18 -51
  6. package/dist/cjs/extract-hast/index.js +4 -3
  7. package/dist/cjs/extract-hast/readability.js +28 -61
  8. package/dist/cjs/extract-hast/strongly.js +2 -1
  9. package/dist/cjs/extract-hast/utils.js +0 -15
  10. package/dist/cjs/html-to-mdast.js +6 -6
  11. package/dist/cjs/link-replacer.js +10 -5
  12. package/dist/cjs/loaders/playwright.js +36 -7
  13. package/dist/cjs/mdast-handlers/custom-br-handler.js +31 -0
  14. package/dist/cjs/mdast-to-markdown.js +1 -1
  15. package/dist/cjs/utils/hast-utils.js +71 -0
  16. package/dist/cli/bin.js +3 -4
  17. package/dist/cli/commands/webforai/index.js +53 -0
  18. package/dist/cli/commands/webforai/loadHtml.js +60 -0
  19. package/dist/cli/utils.js +18 -51
  20. package/dist/extract-hast/index.js +4 -3
  21. package/dist/extract-hast/readability.js +29 -62
  22. package/dist/extract-hast/strongly.js +2 -1
  23. package/dist/extract-hast/utils.js +0 -14
  24. package/dist/html-to-mdast.js +6 -6
  25. package/dist/link-replacer.js +10 -5
  26. package/dist/loaders/playwright.js +36 -7
  27. package/dist/mdast-handlers/custom-br-handler.js +8 -0
  28. package/dist/mdast-to-markdown.js +1 -1
  29. package/dist/types/cli/bin.js.map +1 -1
  30. package/dist/types/cli/commands/webforai/index.d.ts +6 -0
  31. package/dist/types/cli/commands/webforai/index.js.map +1 -0
  32. package/dist/types/cli/commands/webforai/loadHtml.d.ts +3 -0
  33. package/dist/types/cli/commands/webforai/loadHtml.js.map +1 -0
  34. package/dist/types/cli/utils.d.ts +27 -0
  35. package/dist/types/cli/utils.js.map +1 -1
  36. package/dist/types/extract-hast/index.d.ts +9 -4
  37. package/dist/types/extract-hast/index.js.map +1 -1
  38. package/dist/types/extract-hast/readability.d.ts +2 -1
  39. package/dist/types/extract-hast/readability.js.map +1 -1
  40. package/dist/types/extract-hast/strongly.d.ts +2 -1
  41. package/dist/types/extract-hast/strongly.js.map +1 -1
  42. package/dist/types/extract-hast/utils.d.ts +0 -1
  43. package/dist/types/extract-hast/utils.js.map +1 -1
  44. package/dist/types/html-to-mdast.d.ts +4 -0
  45. package/dist/types/html-to-mdast.js.map +1 -1
  46. package/dist/types/link-replacer.d.ts +1 -1
  47. package/dist/types/link-replacer.js.map +1 -1
  48. package/dist/types/loaders/playwright.d.ts +8 -2
  49. package/dist/types/loaders/playwright.js.map +1 -1
  50. package/dist/types/mdast-handlers/custom-br-handler.d.ts +2 -0
  51. package/dist/types/mdast-handlers/custom-br-handler.js.map +1 -0
  52. package/dist/types/mdast-to-markdown.js.map +1 -1
  53. package/dist/types/utils/hast-utils.d.ts +4 -0
  54. package/dist/types/utils/hast-utils.js.map +1 -0
  55. package/dist/utils/hast-utils.js +46 -0
  56. package/package.json +21 -10
  57. package/dist/cjs/cli/commands/handlers/loadHtml.js +0 -127
  58. package/dist/cjs/html-to-markdown.test.js +0 -118
  59. package/dist/cjs/link-replacer.test.js +0 -47
  60. package/dist/cjs/loaders/fetch.test.js +0 -9
  61. package/dist/cjs/loaders/playwright.test.js +0 -17
  62. package/dist/cjs/loaders/puppeteer.test.js +0 -17
  63. package/dist/cli/commands/handlers/index.js +0 -51
  64. package/dist/cli/commands/handlers/loadHtml.js +0 -94
  65. package/dist/html-to-markdown.test.js +0 -118
  66. package/dist/link-replacer.test.js +0 -47
  67. package/dist/loaders/fetch.test.js +0 -9
  68. package/dist/loaders/playwright.test.js +0 -17
  69. package/dist/loaders/puppeteer.test.js +0 -17
  70. package/dist/types/cli/commands/handlers/index.d.ts +0 -13
  71. package/dist/types/cli/commands/handlers/index.js.map +0 -1
  72. package/dist/types/cli/commands/handlers/loadHtml.d.ts +0 -2
  73. package/dist/types/cli/commands/handlers/loadHtml.js.map +0 -1
@@ -1,118 +0,0 @@
1
- "use strict";
2
- var import_fastest_levenshtein = require("fastest-levenshtein");
3
- var import_vitest = require("vitest");
4
- var import_html_to_markdown = require("./html-to-markdown");
5
- var import_fetch = require("./loaders/fetch");
6
- const html = `
7
- <h1>Hello, world!</h1>
8
- <p>This is a paragraph.</p>
9
- <a href="/example.html">Example</a>
10
- <img src="/example.jpg" alt="Example" />
11
- <ul>
12
- <li>Item 1</li>
13
- <li>Item 2</li>
14
- </ul>
15
- `;
16
- const expected = `# Hello, world!
17
-
18
- This is a paragraph.
19
-
20
- [Example](/example.html)
21
-
22
- ![Example](/example.jpg)
23
-
24
- * Item 1
25
- * Item 2
26
- `;
27
- const baseUrlReplaced = `# Hello, world!
28
-
29
- This is a paragraph.
30
-
31
- [Example](https://example.com/example.html)
32
-
33
- ![Example](https://example.com/example.jpg)
34
-
35
- * Item 1
36
- * Item 2
37
- `;
38
- const linkAsText = `# Hello, world!
39
-
40
- This is a paragraph.
41
-
42
- Example
43
-
44
- ![Example](/example.jpg)
45
-
46
- - Item 1
47
- - Item 2
48
- `;
49
- const imageHidden = `# Hello, world!
50
-
51
- This is a paragraph.
52
-
53
- [Example](/example.html)
54
-
55
- - Item 1
56
- - Item 2
57
- `;
58
- const htmlTable = `
59
- <table>
60
- <tr>
61
- <th>Header 1</th>
62
- <th>Header 2</th>
63
- </tr>
64
- <tr>
65
- <td>Cell 1</td>
66
- <td>Cell 2</td>
67
- </tr>
68
- </table>
69
- `;
70
- const expectedTableMarkdown = `
71
- | Header 1 | Header 2 |
72
- | -------- | -------- |
73
- | Cell 1 | Cell 2 |
74
- `;
75
- const expectedTableText = `Header 1 Header 2
76
- Cell 1 Cell 2`;
77
- (0, import_vitest.describe)("htmlToMarkdown", () => {
78
- (0, import_vitest.it)("should convert HTML to Markdown", () => {
79
- const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html, { extractors: false });
80
- const d = (0, import_fastest_levenshtein.distance)(markdown, expected);
81
- (0, import_vitest.expect)(d).lte(5);
82
- });
83
- (0, import_vitest.it)("should convert HTML to Markdown with replaced base URL", () => {
84
- const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html, { baseUrl: "https://example.com", extractors: false });
85
- const d = (0, import_fastest_levenshtein.distance)(markdown, baseUrlReplaced);
86
- (0, import_vitest.expect)(d).lte(5);
87
- });
88
- (0, import_vitest.it)("should convert HTML to Markdown with links as text", () => {
89
- const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html, { linkAsText: true, extractors: false });
90
- const d = (0, import_fastest_levenshtein.distance)(markdown, linkAsText);
91
- (0, import_vitest.expect)(d).lte(5);
92
- });
93
- (0, import_vitest.it)("should convert HTML to Markdown with hidden images", () => {
94
- const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html, { hideImage: true, extractors: false });
95
- const d = (0, import_fastest_levenshtein.distance)(markdown, imageHidden);
96
- (0, import_vitest.expect)(d).lte(5);
97
- });
98
- (0, import_vitest.it)("should convert HTML table to Markdown table", () => {
99
- const markdown = (0, import_html_to_markdown.htmlToMarkdown)(htmlTable, { extractors: false });
100
- const d = (0, import_fastest_levenshtein.distance)(markdown, expectedTableMarkdown);
101
- (0, import_vitest.expect)(d).lte(5);
102
- });
103
- (0, import_vitest.it)("should convert HTML table with table as text option", () => {
104
- const markdown = (0, import_html_to_markdown.htmlToMarkdown)(htmlTable, { tableAsText: true, extractors: false });
105
- const d = (0, import_fastest_levenshtein.distance)(markdown, expectedTableText);
106
- (0, import_vitest.expect)(d).lte(10);
107
- });
108
- });
109
- (0, import_vitest.describe)("htmlToMarkdown E2E", () => {
110
- (0, import_vitest.it)("Converting for good", async () => {
111
- const html1 = await (0, import_fetch.loadHtml)("https://www.npmjs.com/package/webforai");
112
- const markdown1 = (0, import_html_to_markdown.htmlToMarkdown)(html1, { linkAsText: true, hideImage: true });
113
- const html2 = await (0, import_fetch.loadHtml)("https://github.com/inaridiy/webforai");
114
- const markdown2 = (0, import_html_to_markdown.htmlToMarkdown)(html2, { linkAsText: true, hideImage: true });
115
- const d = (0, import_fastest_levenshtein.distance)(markdown1, markdown2);
116
- (0, import_vitest.expect)(d).lte(2500);
117
- });
118
- });
@@ -1,47 +0,0 @@
1
- "use strict";
2
- var import_vitest = require("vitest");
3
- var import_link_replacer = require("./link-replacer");
4
- const markdown = `# Hello, world!
5
-
6
- This is a paragraph.
7
-
8
- [Example](/example.html)
9
-
10
- ![Example](/example.jpg)
11
-
12
- [Absolute Link](https://www.google.com)
13
-
14
- [Link with hash](/page#hash)
15
-
16
- [Link with query](/page?query=string)`;
17
- const expected = `# Hello, world!
18
-
19
- This is a paragraph.
20
-
21
- [Example](https://example.com/example.html)
22
-
23
- ![Example](https://example.com/example.jpg)
24
-
25
- [Absolute Link](https://www.google.com)
26
-
27
- [Link with hash](https://example.com/page#hash)
28
-
29
- [Link with query](https://example.com/page?query=string)`;
30
- (0, import_vitest.describe)("linkReplacer", () => {
31
- (0, import_vitest.it)("should replace relative links", () => {
32
- const replaced = (0, import_link_replacer.linkReplacer)(markdown, "https://example.com");
33
- (0, import_vitest.expect)(replaced).toEqual(expected);
34
- });
35
- (0, import_vitest.it)("should not replace absolute links", () => {
36
- const replaced = (0, import_link_replacer.linkReplacer)("[Absolute Link](https://www.google.com)", "https://example.com");
37
- (0, import_vitest.expect)(replaced).toEqual("[Absolute Link](https://www.google.com)");
38
- });
39
- (0, import_vitest.it)("should handle links with hashes", () => {
40
- const replaced = (0, import_link_replacer.linkReplacer)("[Link with hash](/page#hash)", "https://example.com");
41
- (0, import_vitest.expect)(replaced).toEqual("[Link with hash](https://example.com/page#hash)");
42
- });
43
- (0, import_vitest.it)("should handle links with query parameters", () => {
44
- const replaced = (0, import_link_replacer.linkReplacer)("[Link with query](/page?query=string)", "https://example.com");
45
- (0, import_vitest.expect)(replaced).toEqual("[Link with query](https://example.com/page?query=string)");
46
- });
47
- });
@@ -1,9 +0,0 @@
1
- "use strict";
2
- var import_vitest = require("vitest");
3
- var import_fetch = require("./fetch");
4
- (0, import_vitest.describe)("Fetch loader", () => {
5
- (0, import_vitest.it)("should load the HTML of a URL", async () => {
6
- const html = await (0, import_fetch.loadHtml)("https://example.com");
7
- (0, import_vitest.expect)(html).toContain("Example Domain");
8
- });
9
- });
@@ -1,17 +0,0 @@
1
- "use strict";
2
- var import_playwright = require("playwright");
3
- var import_vitest = require("vitest");
4
- var import_playwright2 = require("./playwright");
5
- (0, import_vitest.describe)("Playwright loader", () => {
6
- (0, import_vitest.it)("should load the HTML of a URL", async () => {
7
- const html = await (0, import_playwright2.loadHtml)("https://example.com");
8
- (0, import_vitest.expect)(html).toContain("Example Domain");
9
- });
10
- (0, import_vitest.it)("should load the HTML of a URL using a custom context", async () => {
11
- const context = await import_playwright.chromium.launch({ headless: true });
12
- const html = await (0, import_playwright2.loadHtml)("https://example.com", context);
13
- (0, import_vitest.expect)(html).toContain("Example Domain");
14
- (0, import_vitest.expect)(context.isConnected()).toBe(true);
15
- await context.close();
16
- });
17
- });
@@ -1,17 +0,0 @@
1
- "use strict";
2
- var import_playwright = require("playwright");
3
- var import_vitest = require("vitest");
4
- var import_puppeteer = require("./puppeteer");
5
- (0, import_vitest.describe)("Puppeteer loader", () => {
6
- (0, import_vitest.it)("should load the HTML of a URL", async () => {
7
- const html = await (0, import_puppeteer.loadHtml)("https://example.com");
8
- (0, import_vitest.expect)(html).toContain("Example Domain");
9
- });
10
- (0, import_vitest.it)("should load the HTML of a URL using a custom puppeteer context", async () => {
11
- const context = await import_playwright.chromium.launch({ headless: true });
12
- const html = await (0, import_puppeteer.loadHtml)("https://example.com", { headless: true });
13
- (0, import_vitest.expect)(html).toContain("Example Domain");
14
- (0, import_vitest.expect)(context.isConnected()).toBe(true);
15
- await context.close();
16
- });
17
- });
@@ -1,51 +0,0 @@
1
- // src/cli/commands/handlers/index.ts
2
- import fs from "node:fs";
3
- import path from "node:path";
4
- import { outro } from "@clack/prompts";
5
- import { htmlToMarkdown } from "../../../html-to-markdown.js";
6
- import { inputOutputPath } from "../../helpers/inputOutputPath.js";
7
- import { inputSourcePath } from "../../helpers/inputSourcePath.js";
8
- import { selectExtractMode } from "../../helpers/selectExtractMode.js";
9
- import { selectLoader } from "../../helpers/selectLoader.js";
10
- import { isUrl } from "../../utils.js";
11
- import { loadHtml } from "./loadHtml.js";
12
- var aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
13
- var readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
14
- var webforaiCommand = async (initialPath, initialOutputPath, options) => {
15
- const sourcePath = initialPath ?? await inputSourcePath();
16
- if (options.debug) {
17
- console.debug(`sourcePath: ${sourcePath}`);
18
- }
19
- const loader = isUrl(sourcePath) ? { type: "web", loader: options.loader ?? await selectLoader() } : { type: "local", loader: null };
20
- if (options.debug) {
21
- console.debug(`loader: ${loader}`);
22
- }
23
- const outputPath = initialOutputPath ?? await inputOutputPath(sourcePath);
24
- if (options.debug) {
25
- console.debug(`outputPath: ${outputPath}`);
26
- }
27
- const mode = options.mode ?? await selectExtractMode();
28
- if (options.debug) {
29
- console.debug(`mode: ${mode}`);
30
- }
31
- const content = await loadHtml(sourcePath, loader);
32
- if (options.debug) {
33
- console.debug(`content: ${content}`);
34
- }
35
- const markdown = htmlToMarkdown(content, {
36
- baseUrl: isUrl(sourcePath) ? sourcePath : options.baseUrl,
37
- ...mode === "ai" ? aiModeOptions : readabilityModeOptions
38
- });
39
- if (options.debug) {
40
- console.debug(`markdown: ${markdown}`);
41
- }
42
- const directory = path.dirname(outputPath);
43
- if (!fs.existsSync(directory)) {
44
- fs.mkdirSync(directory, { recursive: true });
45
- }
46
- fs.writeFileSync(outputPath, markdown);
47
- outro("Markdown conversion is complete!");
48
- };
49
- export {
50
- webforaiCommand
51
- };
@@ -1,94 +0,0 @@
1
- // src/cli/commands/handlers/loadHtml.ts
2
- import fs from "node:fs/promises";
3
- import { cancel, spinner as clackSpinner, confirm, spinner } from "@clack/prompts";
4
- import pc from "picocolors";
5
- import { $ } from "zx";
6
- var loadHtml = async (sourcePath, loader) => {
7
- const loadSpinner = clackSpinner();
8
- try {
9
- switch (loader.type) {
10
- case "local":
11
- return await loadLocalHtml(sourcePath, loadSpinner);
12
- case "web":
13
- switch (loader.loader) {
14
- case "fetch":
15
- return await loadFetchHtml(sourcePath, loadSpinner);
16
- case "playwright":
17
- return await loadPlaywrightHtml(sourcePath, loadSpinner);
18
- default:
19
- throw new Error("Invalid loader");
20
- }
21
- default:
22
- throw new Error("Invalid loader type");
23
- }
24
- } catch (error) {
25
- loadSpinner.stop(pc.red("Content loading failed!"));
26
- console.error(error);
27
- process.exit(1);
28
- }
29
- };
30
- var loadLocalHtml = async (sourcePath, loadSpinner) => {
31
- loadSpinner.start("Retrieving content.");
32
- try {
33
- await fs.access(sourcePath);
34
- } catch {
35
- throw new Error("The file does not exist");
36
- }
37
- const stats = await fs.stat(sourcePath);
38
- if (!stats.isFile()) {
39
- throw new Error("The source must be a file");
40
- }
41
- const content = await fs.readFile(sourcePath, "utf-8");
42
- loadSpinner.stop("Content retrieval is complete!");
43
- return content;
44
- };
45
- var loadFetchHtml = async (sourcePath, loadSpinner) => {
46
- loadSpinner.start("Downloading content.");
47
- const { loadHtml: loadHtml2 } = await import("../../../loaders/fetch.js");
48
- const content = await loadHtml2(sourcePath);
49
- loadSpinner.stop("Content download is complete!");
50
- return content;
51
- };
52
- var loadPlaywrightHtml = async (sourcePath, loadSpinner) => {
53
- const launchSpinner = clackSpinner();
54
- launchSpinner.start("Launching playwright.");
55
- const playwright = await import("playwright");
56
- let browser;
57
- try {
58
- browser = await playwright.chromium.launch();
59
- launchSpinner.stop("Playwright launched!");
60
- } catch (error) {
61
- await handlePlaywrightLaunchError(error);
62
- browser = await playwright.chromium.launch();
63
- }
64
- loadSpinner.start("Loading content.");
65
- const { loadHtml: loadHtml2 } = await import("../../../loaders/playwright.js");
66
- const content = await loadHtml2(sourcePath, browser);
67
- loadSpinner.stop("Content download is complete!");
68
- await browser.close();
69
- return content;
70
- };
71
- var handlePlaywrightLaunchError = async (error) => {
72
- if (!error.message.includes("npx playwright install")) {
73
- throw error;
74
- }
75
- const isInstall = await confirm({
76
- message: "To continue, you will need to download a browser. Do you wish to continue?"
77
- });
78
- if (!isInstall) {
79
- cancel("End of process.");
80
- process.exit(1);
81
- }
82
- const installDepsSpinner = spinner();
83
- installDepsSpinner.start("Installing browser dependencies.");
84
- try {
85
- await $`npx playwright install`;
86
- installDepsSpinner.stop("Browser dependencies installed!");
87
- } catch (error2) {
88
- installDepsSpinner.stop(pc.red("Browser dependencies failed to install!"));
89
- throw error2;
90
- }
91
- };
92
- export {
93
- loadHtml
94
- };
@@ -1,118 +0,0 @@
1
- // src/html-to-markdown.test.ts
2
- import { distance } from "fastest-levenshtein";
3
- import { describe, expect, it } from "vitest";
4
- import { htmlToMarkdown } from "./html-to-markdown.js";
5
- import { loadHtml } from "./loaders/fetch.js";
6
- var html = `
7
- <h1>Hello, world!</h1>
8
- <p>This is a paragraph.</p>
9
- <a href="/example.html">Example</a>
10
- <img src="/example.jpg" alt="Example" />
11
- <ul>
12
- <li>Item 1</li>
13
- <li>Item 2</li>
14
- </ul>
15
- `;
16
- var expected = `# Hello, world!
17
-
18
- This is a paragraph.
19
-
20
- [Example](/example.html)
21
-
22
- ![Example](/example.jpg)
23
-
24
- * Item 1
25
- * Item 2
26
- `;
27
- var baseUrlReplaced = `# Hello, world!
28
-
29
- This is a paragraph.
30
-
31
- [Example](https://example.com/example.html)
32
-
33
- ![Example](https://example.com/example.jpg)
34
-
35
- * Item 1
36
- * Item 2
37
- `;
38
- var linkAsText = `# Hello, world!
39
-
40
- This is a paragraph.
41
-
42
- Example
43
-
44
- ![Example](/example.jpg)
45
-
46
- - Item 1
47
- - Item 2
48
- `;
49
- var imageHidden = `# Hello, world!
50
-
51
- This is a paragraph.
52
-
53
- [Example](/example.html)
54
-
55
- - Item 1
56
- - Item 2
57
- `;
58
- var htmlTable = `
59
- <table>
60
- <tr>
61
- <th>Header 1</th>
62
- <th>Header 2</th>
63
- </tr>
64
- <tr>
65
- <td>Cell 1</td>
66
- <td>Cell 2</td>
67
- </tr>
68
- </table>
69
- `;
70
- var expectedTableMarkdown = `
71
- | Header 1 | Header 2 |
72
- | -------- | -------- |
73
- | Cell 1 | Cell 2 |
74
- `;
75
- var expectedTableText = `Header 1 Header 2
76
- Cell 1 Cell 2`;
77
- describe("htmlToMarkdown", () => {
78
- it("should convert HTML to Markdown", () => {
79
- const markdown = htmlToMarkdown(html, { extractors: false });
80
- const d = distance(markdown, expected);
81
- expect(d).lte(5);
82
- });
83
- it("should convert HTML to Markdown with replaced base URL", () => {
84
- const markdown = htmlToMarkdown(html, { baseUrl: "https://example.com", extractors: false });
85
- const d = distance(markdown, baseUrlReplaced);
86
- expect(d).lte(5);
87
- });
88
- it("should convert HTML to Markdown with links as text", () => {
89
- const markdown = htmlToMarkdown(html, { linkAsText: true, extractors: false });
90
- const d = distance(markdown, linkAsText);
91
- expect(d).lte(5);
92
- });
93
- it("should convert HTML to Markdown with hidden images", () => {
94
- const markdown = htmlToMarkdown(html, { hideImage: true, extractors: false });
95
- const d = distance(markdown, imageHidden);
96
- expect(d).lte(5);
97
- });
98
- it("should convert HTML table to Markdown table", () => {
99
- const markdown = htmlToMarkdown(htmlTable, { extractors: false });
100
- const d = distance(markdown, expectedTableMarkdown);
101
- expect(d).lte(5);
102
- });
103
- it("should convert HTML table with table as text option", () => {
104
- const markdown = htmlToMarkdown(htmlTable, { tableAsText: true, extractors: false });
105
- const d = distance(markdown, expectedTableText);
106
- expect(d).lte(10);
107
- });
108
- });
109
- describe("htmlToMarkdown E2E", () => {
110
- it("Converting for good", async () => {
111
- const html1 = await loadHtml("https://www.npmjs.com/package/webforai");
112
- const markdown1 = htmlToMarkdown(html1, { linkAsText: true, hideImage: true });
113
- const html2 = await loadHtml("https://github.com/inaridiy/webforai");
114
- const markdown2 = htmlToMarkdown(html2, { linkAsText: true, hideImage: true });
115
- const d = distance(markdown1, markdown2);
116
- expect(d).lte(2500);
117
- });
118
- });
@@ -1,47 +0,0 @@
1
- // src/link-replacer.test.ts
2
- import { describe, expect, it } from "vitest";
3
- import { linkReplacer } from "./link-replacer.js";
4
- var markdown = `# Hello, world!
5
-
6
- This is a paragraph.
7
-
8
- [Example](/example.html)
9
-
10
- ![Example](/example.jpg)
11
-
12
- [Absolute Link](https://www.google.com)
13
-
14
- [Link with hash](/page#hash)
15
-
16
- [Link with query](/page?query=string)`;
17
- var expected = `# Hello, world!
18
-
19
- This is a paragraph.
20
-
21
- [Example](https://example.com/example.html)
22
-
23
- ![Example](https://example.com/example.jpg)
24
-
25
- [Absolute Link](https://www.google.com)
26
-
27
- [Link with hash](https://example.com/page#hash)
28
-
29
- [Link with query](https://example.com/page?query=string)`;
30
- describe("linkReplacer", () => {
31
- it("should replace relative links", () => {
32
- const replaced = linkReplacer(markdown, "https://example.com");
33
- expect(replaced).toEqual(expected);
34
- });
35
- it("should not replace absolute links", () => {
36
- const replaced = linkReplacer("[Absolute Link](https://www.google.com)", "https://example.com");
37
- expect(replaced).toEqual("[Absolute Link](https://www.google.com)");
38
- });
39
- it("should handle links with hashes", () => {
40
- const replaced = linkReplacer("[Link with hash](/page#hash)", "https://example.com");
41
- expect(replaced).toEqual("[Link with hash](https://example.com/page#hash)");
42
- });
43
- it("should handle links with query parameters", () => {
44
- const replaced = linkReplacer("[Link with query](/page?query=string)", "https://example.com");
45
- expect(replaced).toEqual("[Link with query](https://example.com/page?query=string)");
46
- });
47
- });
@@ -1,9 +0,0 @@
1
- // src/loaders/fetch.test.ts
2
- import { describe, expect, it } from "vitest";
3
- import { loadHtml } from "./fetch.js";
4
- describe("Fetch loader", () => {
5
- it("should load the HTML of a URL", async () => {
6
- const html = await loadHtml("https://example.com");
7
- expect(html).toContain("Example Domain");
8
- });
9
- });
@@ -1,17 +0,0 @@
1
- // src/loaders/playwright.test.ts
2
- import { chromium } from "playwright";
3
- import { describe, expect, it } from "vitest";
4
- import { loadHtml } from "./playwright.js";
5
- describe("Playwright loader", () => {
6
- it("should load the HTML of a URL", async () => {
7
- const html = await loadHtml("https://example.com");
8
- expect(html).toContain("Example Domain");
9
- });
10
- it("should load the HTML of a URL using a custom context", async () => {
11
- const context = await chromium.launch({ headless: true });
12
- const html = await loadHtml("https://example.com", context);
13
- expect(html).toContain("Example Domain");
14
- expect(context.isConnected()).toBe(true);
15
- await context.close();
16
- });
17
- });
@@ -1,17 +0,0 @@
1
- // src/loaders/puppeteer.test.ts
2
- import { chromium } from "playwright.js";
3
- import { describe, expect, it } from "vitest";
4
- import { loadHtml } from "./puppeteer.js";
5
- describe("Puppeteer loader", () => {
6
- it("should load the HTML of a URL", async () => {
7
- const html = await loadHtml("https://example.com");
8
- expect(html).toContain("Example Domain");
9
- });
10
- it("should load the HTML of a URL using a custom puppeteer context", async () => {
11
- const context = await chromium.launch({ headless: true });
12
- const html = await loadHtml("https://example.com", { headless: true });
13
- expect(html).toContain("Example Domain");
14
- expect(context.isConnected()).toBe(true);
15
- await context.close();
16
- });
17
- });
@@ -1,13 +0,0 @@
1
- export type LoaderOptions = {
2
- type: "web";
3
- loader: "playwright" | "pw" | "fetch" | string;
4
- } | {
5
- type: "local";
6
- loader: null;
7
- };
8
- export declare const webforaiCommand: (initialPath: string, initialOutputPath: string, options: {
9
- mode?: string;
10
- loader: string;
11
- debug?: boolean;
12
- baseUrl?: string;
13
- }) => Promise<void>;
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/cli/commands/handlers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,MAAM,aAAa,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AAC/E,MAAM,sBAAsB,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAM3F,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EACnC,WAAmB,EACnB,iBAAyB,EACzB,OAA6E,EAC5E,EAAE;IACH,MAAM,UAAU,GAAG,WAAW,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;IAC5D,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,MAAM,GAAkB,KAAK,CAAC,UAAU,CAAC;QAC9C,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,YAAY,EAAE,CAAC,EAAE;QACnE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,WAAW,MAAM,EAAE,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,UAAU,GAAG,iBAAiB,IAAI,CAAC,MAAM,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;IAC5E,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC;IACzD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACnD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,EAAE;QACxC,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO;QACzD,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,sBAAsB,CAAC;KAC3D,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAE3C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IACD,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAEvC,KAAK,CAAC,kCAAkC,CAAC,CAAC;AAC3C,CAAC,CAAC"}
@@ -1,2 +0,0 @@
1
- import type { LoaderOptions } from "./index";
2
- export declare const loadHtml: (sourcePath: string, loader: LoaderOptions) => Promise<string>;
@@ -1 +0,0 @@
1
- {"version":3,"file":"loadHtml.js","sourceRoot":"","sources":["../../../../../src/cli/commands/handlers/loadHtml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EAAE,MAAM,EAAE,OAAO,IAAI,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACnF,OAAO,EAAE,MAAM,YAAY,CAAC;AAE5B,OAAO,EAAE,CAAC,EAAE,MAAM,IAAI,CAAC;AAKvB,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,UAAkB,EAAE,MAAqB,EAAmB,EAAE;IAC5F,MAAM,WAAW,GAAG,YAAY,EAAE,CAAC;IAEnC,IAAI,CAAC;QACJ,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,OAAO;gBACX,OAAO,MAAM,aAAa,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;YACrD,KAAK,KAAK;gBACT,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;oBACvB,KAAK,OAAO;wBACX,OAAO,MAAM,aAAa,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;oBACrD,KAAK,YAAY;wBAChB,OAAO,MAAM,kBAAkB,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;oBAC1D;wBACC,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;gBACpC,CAAC;YACF;gBACC,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,CAAC;QACpD,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,KAAK,EAAE,UAAkB,EAAE,WAAwB,EAAmB,EAAE;IAC7F,WAAW,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,IAAI,CAAC;QACJ,MAAM,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACvD,WAAW,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IACnD,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,KAAK,EAAE,UAAkB,EAAE,WAAwB,EAAmB,EAAE;IAC7F,WAAW,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC3C,WAAW,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,KAAK,EAAE,UAAkB,EAAE,WAAwB,EAAmB,EAAE;IAClG,MAAM,aAAa,GAAG,YAAY,EAAE,CAAC;IACrC,aAAa,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IAE7C,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAC9C,IAAI,OAA0B,CAAC;IAE/B,IAAI,CAAC;QACJ,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC7C,aAAa,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,2BAA2B,CAAC,KAAc,CAAC,CAAC;QAClD,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;IAC9C,CAAC;IAED,WAAW,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACtC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAElD,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IAEtB,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,2BAA2B,GAAG,KAAK,EAAE,KAAY,EAAiB,EAAE;IACzE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;QACvD,MAAM,KAAK,CAAC;IACb,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC;QAC/B,OAAO,EAAE,4EAA4E;KACrF,CAAC,CAAC;IACH,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAC1B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,kBAAkB,GAAG,OAAO,EAAE,CAAC;IACrC,kBAAkB,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAE7D,IAAI,CAAC;QACJ,MAAM,CAAC,CAAA,wBAAwB,CAAC;QAChC,kBAAkB,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAC5D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC,CAAC;QAC3E,MAAM,KAAK,CAAC;IACb,CAAC;AACF,CAAC,CAAC"}