webforai 1.3.2 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/extract-hast/readability.js +5 -1
- package/dist/cjs/html-to-markdown.test.js +3 -2
- package/dist/cjs/loaders/fetch.js +34 -0
- package/dist/cjs/loaders/fetch.test.js +9 -0
- package/dist/cjs/loaders/playwright.js +4 -13
- package/dist/cjs/loaders/playwright.test.js +17 -0
- package/dist/extract-hast/readability.js +5 -1
- package/dist/html-to-markdown.test.js +3 -2
- package/dist/loaders/fetch.js +10 -0
- package/dist/loaders/fetch.test.js +9 -0
- package/dist/loaders/playwright.js +4 -13
- package/dist/loaders/playwright.test.js +17 -0
- package/dist/types/extract-hast/readability.js.map +1 -1
- package/dist/types/loaders/fetch.d.ts +9 -0
- package/dist/types/loaders/fetch.js.map +1 -0
- package/dist/types/loaders/playwright.d.ts +16 -2
- package/dist/types/loaders/playwright.js.map +1 -1
- package/package.json +6 -1
|
@@ -30,7 +30,8 @@ const UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert
|
|
|
30
30
|
const REGEXPS = {
|
|
31
31
|
hidden: /hidden|invisible|fallback-image/i,
|
|
32
32
|
byline: /byline|author|dateline|writtenby|p-author/i,
|
|
33
|
-
|
|
33
|
+
specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
|
|
34
|
+
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore/i,
|
|
34
35
|
okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
|
|
35
36
|
};
|
|
36
37
|
const BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry", "main"];
|
|
@@ -96,6 +97,9 @@ const unlikelyElementFilter = (node) => {
|
|
|
96
97
|
return true;
|
|
97
98
|
}
|
|
98
99
|
const match = (0, import_utils.matchString)(element);
|
|
100
|
+
if (REGEXPS.specialUnlikelyCandidates.test(match)) {
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
99
103
|
if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
|
|
100
104
|
return false;
|
|
101
105
|
}
|
|
@@ -24,6 +24,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
24
24
|
var import_fastest_levenshtein = require("fastest-levenshtein");
|
|
25
25
|
var import_vitest = require("vitest");
|
|
26
26
|
var import_html_to_markdown = require("./html-to-markdown");
|
|
27
|
+
var import_fetch = require("./loaders/fetch");
|
|
27
28
|
const html = `
|
|
28
29
|
<h1>Hello, world!</h1>
|
|
29
30
|
<p>This is a paragraph.</p>
|
|
@@ -129,14 +130,14 @@ Cell 1 Cell 2`;
|
|
|
129
130
|
});
|
|
130
131
|
(0, import_vitest.describe)("htmlToMarkdown E2E", () => {
|
|
131
132
|
(0, import_vitest.it)("should convert npm README to Markdown ", async () => {
|
|
132
|
-
const html2 = await
|
|
133
|
+
const html2 = await (0, import_fetch.loadHtml)("https://www.npmjs.com/package/webforai");
|
|
133
134
|
const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html2, { baseUrl: "https://www.npmjs.com/package/webforai" });
|
|
134
135
|
const original = await import("../README.md?raw");
|
|
135
136
|
const d = (0, import_fastest_levenshtein.distance)(markdown, original.default);
|
|
136
137
|
(0, import_vitest.expect)(d).lte(400);
|
|
137
138
|
});
|
|
138
139
|
(0, import_vitest.it)("should convert GitHub README to Markdown ", async () => {
|
|
139
|
-
const html2 = await
|
|
140
|
+
const html2 = await (0, import_fetch.loadHtml)("https://github.com/inaridiy/webforai");
|
|
140
141
|
const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html2, { baseUrl: "https://www.npmjs.com/package/webforai" });
|
|
141
142
|
const original = await import("../../../README.md?raw");
|
|
142
143
|
const d = (0, import_fastest_levenshtein.distance)(markdown, original.default);
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
var fetch_exports = {};
|
|
20
|
+
__export(fetch_exports, {
|
|
21
|
+
USER_AGENT: () => USER_AGENT,
|
|
22
|
+
loadHtml: () => loadHtml
|
|
23
|
+
});
|
|
24
|
+
module.exports = __toCommonJS(fetch_exports);
|
|
25
|
+
const USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
|
|
26
|
+
const loadHtml = async (url, userAgent = USER_AGENT) => {
|
|
27
|
+
const response = await fetch(url, { headers: { "User-Agent": userAgent } });
|
|
28
|
+
return response.text();
|
|
29
|
+
};
|
|
30
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
31
|
+
0 && (module.exports = {
|
|
32
|
+
USER_AGENT,
|
|
33
|
+
loadHtml
|
|
34
|
+
});
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var import_vitest = require("vitest");
|
|
3
|
+
var import_fetch = require("./fetch");
|
|
4
|
+
(0, import_vitest.describe)("Fetch loader", () => {
|
|
5
|
+
(0, import_vitest.it)("should load the HTML of a URL", async () => {
|
|
6
|
+
const html = await (0, import_fetch.loadHtml)("https://example.com");
|
|
7
|
+
(0, import_vitest.expect)(html).toContain("Example Domain");
|
|
8
|
+
});
|
|
9
|
+
});
|
|
@@ -22,23 +22,14 @@ __export(playwright_exports, {
|
|
|
22
22
|
});
|
|
23
23
|
module.exports = __toCommonJS(playwright_exports);
|
|
24
24
|
var import_playwright = require("playwright");
|
|
25
|
-
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
26
25
|
const loadHtml = async (url, context) => {
|
|
27
|
-
|
|
28
|
-
let _context;
|
|
29
|
-
if (context) {
|
|
30
|
-
_context = context;
|
|
31
|
-
} else {
|
|
32
|
-
_browser = await import_playwright.chromium.launch({ headless: true });
|
|
33
|
-
_context = await _browser.newContext();
|
|
34
|
-
}
|
|
26
|
+
const _context = context ?? await import_playwright.chromium.launch({ headless: true });
|
|
35
27
|
const page = await _context.newPage();
|
|
36
|
-
await page.goto(url);
|
|
37
|
-
await Promise.race([page.waitForLoadState("networkidle"), sleep(1e4)]);
|
|
28
|
+
await page.goto(url, { waitUntil: "load" });
|
|
38
29
|
const html = await page.content();
|
|
39
30
|
await page.close();
|
|
40
|
-
if (
|
|
41
|
-
await
|
|
31
|
+
if (!context) {
|
|
32
|
+
await _context.close();
|
|
42
33
|
}
|
|
43
34
|
return html;
|
|
44
35
|
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var import_playwright = require("playwright");
|
|
3
|
+
var import_vitest = require("vitest");
|
|
4
|
+
var import_playwright2 = require("./playwright");
|
|
5
|
+
(0, import_vitest.describe)("Playwright loader", () => {
|
|
6
|
+
(0, import_vitest.it)("should load the HTML of a URL", async () => {
|
|
7
|
+
const html = await (0, import_playwright2.loadHtml)("https://example.com");
|
|
8
|
+
(0, import_vitest.expect)(html).toContain("Example Domain");
|
|
9
|
+
});
|
|
10
|
+
(0, import_vitest.it)("should load the HTML of a URL using a custom context", async () => {
|
|
11
|
+
const context = await import_playwright.chromium.launch({ headless: true });
|
|
12
|
+
const html = await (0, import_playwright2.loadHtml)("https://example.com", context);
|
|
13
|
+
(0, import_vitest.expect)(html).toContain("Example Domain");
|
|
14
|
+
(0, import_vitest.expect)(context.isConnected()).toBe(true);
|
|
15
|
+
await context.close();
|
|
16
|
+
});
|
|
17
|
+
});
|
|
@@ -8,7 +8,8 @@ var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert",
|
|
|
8
8
|
var REGEXPS = {
|
|
9
9
|
hidden: /hidden|invisible|fallback-image/i,
|
|
10
10
|
byline: /byline|author|dateline|writtenby|p-author/i,
|
|
11
|
-
|
|
11
|
+
specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
|
|
12
|
+
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore/i,
|
|
12
13
|
okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
|
|
13
14
|
};
|
|
14
15
|
var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry", "main"];
|
|
@@ -74,6 +75,9 @@ var unlikelyElementFilter = (node) => {
|
|
|
74
75
|
return true;
|
|
75
76
|
}
|
|
76
77
|
const match = matchString(element);
|
|
78
|
+
if (REGEXPS.specialUnlikelyCandidates.test(match)) {
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
77
81
|
if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
|
|
78
82
|
return false;
|
|
79
83
|
}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import { distance } from "fastest-levenshtein";
|
|
3
3
|
import { describe, expect, it } from "vitest";
|
|
4
4
|
import { htmlToMarkdown } from "./html-to-markdown.js";
|
|
5
|
+
import { loadHtml } from "./loaders/fetch.js";
|
|
5
6
|
var html = `
|
|
6
7
|
<h1>Hello, world!</h1>
|
|
7
8
|
<p>This is a paragraph.</p>
|
|
@@ -107,14 +108,14 @@ describe("htmlToMarkdown", () => {
|
|
|
107
108
|
});
|
|
108
109
|
describe("htmlToMarkdown E2E", () => {
|
|
109
110
|
it("should convert npm README to Markdown ", async () => {
|
|
110
|
-
const html2 = await
|
|
111
|
+
const html2 = await loadHtml("https://www.npmjs.com/package/webforai");
|
|
111
112
|
const markdown = htmlToMarkdown(html2, { baseUrl: "https://www.npmjs.com/package/webforai" });
|
|
112
113
|
const original = await import("../README.md?raw");
|
|
113
114
|
const d = distance(markdown, original.default);
|
|
114
115
|
expect(d).lte(400);
|
|
115
116
|
});
|
|
116
117
|
it("should convert GitHub README to Markdown ", async () => {
|
|
117
|
-
const html2 = await
|
|
118
|
+
const html2 = await loadHtml("https://github.com/inaridiy/webforai");
|
|
118
119
|
const markdown = htmlToMarkdown(html2, { baseUrl: "https://www.npmjs.com/package/webforai" });
|
|
119
120
|
const original = await import("../../../README.md?raw");
|
|
120
121
|
const d = distance(markdown, original.default);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// src/loaders/fetch.ts
|
|
2
|
+
var USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
|
|
3
|
+
var loadHtml = async (url, userAgent = USER_AGENT) => {
|
|
4
|
+
const response = await fetch(url, { headers: { "User-Agent": userAgent } });
|
|
5
|
+
return response.text();
|
|
6
|
+
};
|
|
7
|
+
export {
|
|
8
|
+
USER_AGENT,
|
|
9
|
+
loadHtml
|
|
10
|
+
};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// src/loaders/fetch.test.ts
|
|
2
|
+
import { describe, expect, it } from "vitest";
|
|
3
|
+
import { loadHtml } from "./fetch.js";
|
|
4
|
+
describe("Fetch loader", () => {
|
|
5
|
+
it("should load the HTML of a URL", async () => {
|
|
6
|
+
const html = await loadHtml("https://example.com");
|
|
7
|
+
expect(html).toContain("Example Domain");
|
|
8
|
+
});
|
|
9
|
+
});
|
|
@@ -1,22 +1,13 @@
|
|
|
1
1
|
// src/loaders/playwright.ts
|
|
2
2
|
import { chromium } from "playwright";
|
|
3
|
-
var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
4
3
|
var loadHtml = async (url, context) => {
|
|
5
|
-
|
|
6
|
-
let _context;
|
|
7
|
-
if (context) {
|
|
8
|
-
_context = context;
|
|
9
|
-
} else {
|
|
10
|
-
_browser = await chromium.launch({ headless: true });
|
|
11
|
-
_context = await _browser.newContext();
|
|
12
|
-
}
|
|
4
|
+
const _context = context ?? await chromium.launch({ headless: true });
|
|
13
5
|
const page = await _context.newPage();
|
|
14
|
-
await page.goto(url);
|
|
15
|
-
await Promise.race([page.waitForLoadState("networkidle"), sleep(1e4)]);
|
|
6
|
+
await page.goto(url, { waitUntil: "load" });
|
|
16
7
|
const html = await page.content();
|
|
17
8
|
await page.close();
|
|
18
|
-
if (
|
|
19
|
-
await
|
|
9
|
+
if (!context) {
|
|
10
|
+
await _context.close();
|
|
20
11
|
}
|
|
21
12
|
return html;
|
|
22
13
|
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// src/loaders/playwright.test.ts
|
|
2
|
+
import { chromium } from "playwright";
|
|
3
|
+
import { describe, expect, it } from "vitest";
|
|
4
|
+
import { loadHtml } from "./playwright.js";
|
|
5
|
+
describe("Playwright loader", () => {
|
|
6
|
+
it("should load the HTML of a URL", async () => {
|
|
7
|
+
const html = await loadHtml("https://example.com");
|
|
8
|
+
expect(html).toContain("Example Domain");
|
|
9
|
+
});
|
|
10
|
+
it("should load the HTML of a URL using a custom context", async () => {
|
|
11
|
+
const context = await chromium.launch({ headless: true });
|
|
12
|
+
const html = await loadHtml("https://example.com", context);
|
|
13
|
+
expect(html).toContain("Example Domain");
|
|
14
|
+
expect(context.isConnected()).toBe(true);
|
|
15
|
+
await context.close();
|
|
16
|
+
});
|
|
17
|
+
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"readability.js","sourceRoot":"","sources":["../../../src/extract-hast/readability.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAU9E,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;AAE5G,MAAM,OAAO,GAAG;IACf,MAAM,EAAE,kCAAkC;IAC1C,MAAM,EAAE,4CAA4C;IACpD,kBAAkB,EACjB,
|
|
1
|
+
{"version":3,"file":"readability.js","sourceRoot":"","sources":["../../../src/extract-hast/readability.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAU9E,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;AAE5G,MAAM,OAAO,GAAG;IACf,MAAM,EAAE,kCAAkC;IAC1C,MAAM,EAAE,4CAA4C;IACpD,yBAAyB,EAAE,8BAA8B;IACzD,kBAAkB,EACjB,yQAAyQ;IAC1Q,oBAAoB,EAAE,mDAAmD;CACzE,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;AAE/G,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEpF,MAAM,mBAAmB,GAAG;IAC3B,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;CACP,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,IAAU,EAAE,EAAE;IACjC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO;IACR,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAChC,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAChC,OAAO;IACR,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IAC3E,IAAI,QAAQ,EAAE,CAAC;QACd,OAAO,QAAkB,CAAC;IAC3B,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,IAAU,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEnF,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC7C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,8BAA8B;IAC9B,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yCAAyC;IACzC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC7E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QAChG,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,IAAc,CAAC,EAAE,CAAC;QAC3F,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,6BAA6B;IAC7B,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3E,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAEnC,IAAI,OAAO,CAAC,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,6BAA6B;IAC7B,IAAI,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzF,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE;IACxC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAE7B,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACd,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;IACpE,MAAM,YAAY,GAAG,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;IAEpE,OAAO,WAAW,KAAK,YAAY,IAAI,WAAW,CAAC;AACpD,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,IAAU,EAAE,EAAE;IACxC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACtB,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAQ,EAAE;IAC1D,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;IAE1C,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAA2B,CAAC;IAC5D,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,EAAE;QAChD,IAAI,CAAC,cAAc,CAAC,IAAY,CAAC,EAAE,CAAC;YACnC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,sBAAsB,CAAC,IAAY,CAAC,EAAE,CAAC;YAC3C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,WAAW,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACvC,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAC3C,IAAI,aAAa,GAAG,IAAI,IAAI,mBAAmB,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAwC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACtH,IAAI,QAAQ,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QACrC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,QAAQ,GAAS,WAAW,CAAC;IACjC,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,CAAC;QACnF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAEpC,IAAI,QAAQ,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC1B,SAAS;QACV,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAElE,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QACtD,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;YACvB,SAAS;QACV,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;YACrC,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM;QACP,CAAC;IACF,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3C,IAAI,CAAC,iBAAiB,CAAC,IAAY,CAAC,EAAE,CAAC;YACtC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,qBAAqB,CAAC,IAAY,CAAC,EAAE,CAAC;YAC1C,OAAO,KAAK,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC,CAAS,CAAC;IAEX,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export declare const USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
|
|
2
|
+
/**
|
|
3
|
+
* Useful function for load the HTML of a URL using the Fetch API.
|
|
4
|
+
* **Not recommended** for use in production environments.
|
|
5
|
+
* @param url - The URL to load.
|
|
6
|
+
* @param userAgent - The user agent to use. Default is a Chrome user agent.
|
|
7
|
+
* @returns The HTML content of the URL.
|
|
8
|
+
*/
|
|
9
|
+
export declare const loadHtml: (url: string, userAgent?: string) => Promise<string>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../../src/loaders/fetch.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,UAAU,GACtB,iHAAiH,CAAC;AAEnH;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAE,SAAS,GAAG,UAAU,EAAE,EAAE;IACrE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;IAC5E,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC,CAAC"}
|
|
@@ -1,2 +1,16 @@
|
|
|
1
|
-
import { type BrowserContext } from "playwright";
|
|
2
|
-
|
|
1
|
+
import { type Browser, type BrowserContext } from "playwright";
|
|
2
|
+
/**
|
|
3
|
+
* Useful function for load the HTML of a URL using Playwright.
|
|
4
|
+
* **Not recommended** for use in production environments.
|
|
5
|
+
* @param url - The URL to load.
|
|
6
|
+
* @param context - The Playwright browser context to use. If not provided, a new browser context will be created and closed after loading the URL.
|
|
7
|
+
* @returns The HTML content of the URL.
|
|
8
|
+
* @example
|
|
9
|
+
* ```ts
|
|
10
|
+
* import { loadHtml } from "webforai/loaders/playwright";
|
|
11
|
+
*
|
|
12
|
+
* const html = await loadHtml("https://example.com");
|
|
13
|
+
* console.log(html);
|
|
14
|
+
* ```
|
|
15
|
+
*/
|
|
16
|
+
export declare const loadHtml: (url: string, context?: BrowserContext | Browser) => Promise<string>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../../src/loaders/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqC,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEzE
|
|
1
|
+
{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../../src/loaders/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqC,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEzE;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAE,OAAkC,EAAE,EAAE;IACjF,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAExE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;IACtC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAClC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IAEnB,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "webforai",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.3",
|
|
4
4
|
"description": "A library that provides a web interface for AI",
|
|
5
5
|
"author": "inaridiy",
|
|
6
6
|
"license": "MIT",
|
|
@@ -42,6 +42,11 @@
|
|
|
42
42
|
"import": "./dist/loaders/playwright.js",
|
|
43
43
|
"require": "./dist/cjs/loaders/playwright.js"
|
|
44
44
|
},
|
|
45
|
+
"./loaders/fetch": {
|
|
46
|
+
"types": "./dist/types/loaders/fetch.d.ts",
|
|
47
|
+
"import": "./dist/loaders/fetch.js",
|
|
48
|
+
"require": "./dist/cjs/loaders/fetch.js"
|
|
49
|
+
},
|
|
45
50
|
"./loaders/cf-puppeteer": {
|
|
46
51
|
"types": "./dist/types/loaders/cf-puppeteer.d.ts",
|
|
47
52
|
"import": "./dist/loaders/cf-puppeteer.js",
|