webforai 1.3.3 → 1.4.1

package/dist/cjs/extract-hast/readability.js

@@ -24,7 +24,6 @@ module.exports = __toCommonJS(readability_exports);
 var import_hast_util_select = require("hast-util-select");
 var import_hast_util_to_string = require("hast-util-to-string");
 var import_unist_util_filter = require("unist-util-filter");
-var import_unist_util_parents = require("unist-util-parents");
 var import_utils = require("./utils");
 const UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
 const REGEXPS = {
@@ -135,8 +134,8 @@ const removeEmptyFilter = (node) => {
 const readabilityExtractHast = (hast) => {
   const lang = String((0, import_hast_util_select.select)("html", hast)?.properties.lang || tryGetLang(hast) || "en");
   const body = (0, import_hast_util_select.select)("body", hast) ?? hast;
-  const proxiedHast = (0, import_unist_util_parents.parents)(body);
-  const baseFilterd = (0, import_unist_util_filter.filter)(proxiedHast, (node) => {
+  const bodyText = (0, import_hast_util_to_string.toString)(body);
+  const baseFilterd = (0, import_unist_util_filter.filter)(body, (node) => {
     if (!metadataFilter(node)) {
       return false;
     }
@@ -148,30 +147,31 @@ const readabilityExtractHast = (hast) => {
   if (!baseFilterd) {
     return { type: "root", children: [] };
   }
-  const baseText = (0, import_hast_util_to_string.toString)(baseFilterd);
+  const baseFilterdText = (0, import_hast_util_to_string.toString)(baseFilterd);
+  let [baseTree, baseText] = baseFilterdText.length > bodyText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [body, bodyText];
   let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
   if (baseText.length < minimalLength) {
     minimalLength = Math.max(0, baseText.length - 200);
   }
-  let bodyTree = baseFilterd;
   for (const selector of BODY_SELECTORS) {
-    const body2 = { type: "root", children: (0, import_hast_util_select.selectAll)(selector, baseFilterd) };
-    const bodyText = (0, import_hast_util_to_string.toString)(body2);
-    if (bodyText.length < 25) {
+    const content = { type: "root", children: (0, import_hast_util_select.selectAll)(selector, baseFilterd) };
+    const contentText = (0, import_hast_util_to_string.toString)(content);
+    if (contentText.length < 25) {
       continue;
     }
-    const links = (0, import_hast_util_select.selectAll)("a", body2);
+    const links = (0, import_hast_util_select.selectAll)("a", content);
     const linkText = links.map((link) => (0, import_hast_util_to_string.toString)(link)).join("");
     const linkDensity = linkText.length / bodyText.length;
     if (linkDensity > 0.4) {
       continue;
     }
-    if (bodyText.length > minimalLength) {
-      bodyTree = body2;
+    if (contentText.length > minimalLength) {
+      baseTree = content;
+      baseText = contentText;
       break;
     }
   }
-  const finalTree = (0, import_unist_util_filter.filter)(bodyTree, (node) => {
+  const finalFilteredTree = (0, import_unist_util_filter.filter)(baseTree, (node) => {
     if (!removeEmptyFilter(node)) {
       return false;
     }
@@ -180,6 +180,7 @@ const readabilityExtractHast = (hast) => {
     }
     return true;
   });
+  const finalTree = (0, import_hast_util_to_string.toString)(finalFilteredTree).length > baseText.length / 3 ? finalFilteredTree : baseTree;
   return finalTree;
 };
 // Annotate the CommonJS export names for ESM import in node:

package/dist/cjs/html-to-markdown.test.js

@@ -131,14 +131,14 @@ Cell 1    Cell 2`;
 (0, import_vitest.describe)("htmlToMarkdown E2E", () => {
   (0, import_vitest.it)("should convert npm README to Markdown ", async () => {
     const html2 = await (0, import_fetch.loadHtml)("https://www.npmjs.com/package/webforai");
-    const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html2, { baseUrl: "https://www.npmjs.com/package/webforai" });
+    const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html2);
     const original = await import("../README.md?raw");
     const d = (0, import_fastest_levenshtein.distance)(markdown, original.default);
-    (0, import_vitest.expect)(d).lte(400);
+    (0, import_vitest.expect)(d).lte(200);
   });
   (0, import_vitest.it)("should convert GitHub README to Markdown ", async () => {
     const html2 = await (0, import_fetch.loadHtml)("https://github.com/inaridiy/webforai");
-    const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html2, { baseUrl: "https://www.npmjs.com/package/webforai" });
+    const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html2);
     const original = await import("../../../README.md?raw");
     const d = (0, import_fastest_levenshtein.distance)(markdown, original.default);
     (0, import_vitest.expect)(d).lte(200);

package/dist/cjs/loaders/puppeteer.js

@@ -0,0 +1,53 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var puppeteer_exports = {};
+__export(puppeteer_exports, {
+  loadHtml: () => loadHtml
+});
+module.exports = __toCommonJS(puppeteer_exports);
+var import_puppeteer = __toESM(require("puppeteer"), 1);
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const loadHtml = async (url, ctx) => {
+  const browser = await import_puppeteer.default.launch(
+    ctx || {
+      headless: true,
+      args: ["--no-sandbox", "--disable-setuid-sandbox"]
+    }
+  );
+  const page = await browser.newPage();
+  await page.goto(url);
+  const html = await page.content();
+  await Promise.race([page.waitForNetworkIdle(), sleep(1e4)]);
+  await page.close();
+  return html;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  loadHtml
+});

package/dist/cjs/loaders/puppeteer.test.js

@@ -0,0 +1,17 @@
+"use strict";
+var import_playwright = require("playwright");
+var import_vitest = require("vitest");
+var import_puppeteer = require("./puppeteer");
+(0, import_vitest.describe)("Puppeteer loader", () => {
+  (0, import_vitest.it)("should load the HTML of a URL", async () => {
+    const html = await (0, import_puppeteer.loadHtml)("https://example.com");
+    (0, import_vitest.expect)(html).toContain("Example Domain");
+  });
+  (0, import_vitest.it)("should load the HTML of a URL using a custom puppeteer context", async () => {
+    const context = await import_playwright.chromium.launch({ headless: true });
+    const html = await (0, import_puppeteer.loadHtml)("https://example.com", { headless: true });
+    (0, import_vitest.expect)(html).toContain("Example Domain");
+    (0, import_vitest.expect)(context.isConnected()).toBe(true);
+    await context.close();
+  });
+});

package/dist/extract-hast/readability.js

@@ -2,7 +2,6 @@
 import { select, selectAll } from "hast-util-select";
 import { toString as hastToString } from "hast-util-to-string";
 import { filter } from "unist-util-filter";
-import { parents } from "unist-util-parents";
 import { classnames, hasAncestors, isStrInclude, matchString } from "./utils.js";
 var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
 var REGEXPS = {
@@ -113,8 +112,8 @@ var removeEmptyFilter = (node) => {
 var readabilityExtractHast = (hast) => {
   const lang = String(select("html", hast)?.properties.lang || tryGetLang(hast) || "en");
   const body = select("body", hast) ?? hast;
-  const proxiedHast = parents(body);
-  const baseFilterd = filter(proxiedHast, (node) => {
+  const bodyText = hastToString(body);
+  const baseFilterd = filter(body, (node) => {
     if (!metadataFilter(node)) {
       return false;
     }
@@ -126,30 +125,31 @@ var readabilityExtractHast = (hast) => {
   if (!baseFilterd) {
     return { type: "root", children: [] };
   }
-  const baseText = hastToString(baseFilterd);
+  const baseFilterdText = hastToString(baseFilterd);
+  let [baseTree, baseText] = baseFilterdText.length > bodyText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [body, bodyText];
   let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
   if (baseText.length < minimalLength) {
     minimalLength = Math.max(0, baseText.length - 200);
   }
-  let bodyTree = baseFilterd;
   for (const selector of BODY_SELECTORS) {
-    const body2 = { type: "root", children: selectAll(selector, baseFilterd) };
-    const bodyText = hastToString(body2);
-    if (bodyText.length < 25) {
+    const content = { type: "root", children: selectAll(selector, baseFilterd) };
+    const contentText = hastToString(content);
+    if (contentText.length < 25) {
       continue;
     }
-    const links = selectAll("a", body2);
+    const links = selectAll("a", content);
     const linkText = links.map((link) => hastToString(link)).join("");
     const linkDensity = linkText.length / bodyText.length;
     if (linkDensity > 0.4) {
       continue;
     }
-    if (bodyText.length > minimalLength) {
-      bodyTree = body2;
+    if (contentText.length > minimalLength) {
+      baseTree = content;
+      baseText = contentText;
       break;
     }
   }
-  const finalTree = filter(bodyTree, (node) => {
+  const finalFilteredTree = filter(baseTree, (node) => {
     if (!removeEmptyFilter(node)) {
       return false;
     }
@@ -158,6 +158,7 @@ var readabilityExtractHast = (hast) => {
     }
     return true;
   });
+  const finalTree = hastToString(finalFilteredTree).length > baseText.length / 3 ? finalFilteredTree : baseTree;
   return finalTree;
 };
 export {

package/dist/html-to-markdown.test.js

@@ -109,14 +109,14 @@ describe("htmlToMarkdown", () => {
 describe("htmlToMarkdown E2E", () => {
   it("should convert npm README to Markdown ", async () => {
     const html2 = await loadHtml("https://www.npmjs.com/package/webforai");
-    const markdown = htmlToMarkdown(html2, { baseUrl: "https://www.npmjs.com/package/webforai" });
+    const markdown = htmlToMarkdown(html2);
     const original = await import("../README.md?raw");
     const d = distance(markdown, original.default);
-    expect(d).lte(400);
+    expect(d).lte(200);
   });
   it("should convert GitHub README to Markdown ", async () => {
     const html2 = await loadHtml("https://github.com/inaridiy/webforai");
-    const markdown = htmlToMarkdown(html2, { baseUrl: "https://www.npmjs.com/package/webforai" });
+    const markdown = htmlToMarkdown(html2);
     const original = await import("../../../README.md?raw");
     const d = distance(markdown, original.default);
     expect(d).lte(200);

package/dist/loaders/puppeteer.js

@@ -0,0 +1,20 @@
+// src/loaders/puppeteer.ts
+import puppeteer from "puppeteer";
+var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+var loadHtml = async (url, ctx) => {
+  const browser = await puppeteer.launch(
+    ctx || {
+      headless: true,
+      args: ["--no-sandbox", "--disable-setuid-sandbox"]
+    }
+  );
+  const page = await browser.newPage();
+  await page.goto(url);
+  const html = await page.content();
+  await Promise.race([page.waitForNetworkIdle(), sleep(1e4)]);
+  await page.close();
+  return html;
+};
+export {
+  loadHtml
+};

package/dist/loaders/puppeteer.test.js

@@ -0,0 +1,17 @@
+// src/loaders/puppeteer.test.ts
+import { chromium } from "playwright.js";
+import { describe, expect, it } from "vitest";
+import { loadHtml } from "./puppeteer.js";
+describe("Puppeteer loader", () => {
+  it("should load the HTML of a URL", async () => {
+    const html = await loadHtml("https://example.com");
+    expect(html).toContain("Example Domain");
+  });
+  it("should load the HTML of a URL using a custom puppeteer context", async () => {
+    const context = await chromium.launch({ headless: true });
+    const html = await loadHtml("https://example.com", { headless: true });
+    expect(html).toContain("Example Domain");
+    expect(context.isConnected()).toBe(true);
+    await context.close();
+  });
+});

package/dist/types/extract-hast/readability.d.ts

@@ -1,11 +1,2 @@
 import type { Nodes as Hast } from "hast";
-type ProxiedHast = Hast & {
-    parent: ProxiedHast | null;
-};
-declare module "hast" {
-    interface Element {
-        parent: ProxiedHast | null;
-    }
-}
 export declare const readabilityExtractHast: (hast: Hast) => Hast;
-export {};

package/dist/types/extract-hast/readability.js.map

@@ -1 +1 @@
-{"version":3,"file":"readability.js","sourceRoot":"","sources":["../../../src/extract-hast/readability.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAU9E,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;AAE5G,MAAM,OAAO,GAAG;IACf,MAAM,EAAE,kCAAkC;IAC1C,MAAM,EAAE,4CAA4C;IACpD,yBAAyB,EAAE,8BAA8B;IACzD,kBAAkB,EACjB,yQAAyQ;IAC1Q,oBAAoB,EAAE,mDAAmD;CACzE,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;AAE/G,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEpF,MAAM,mBAAmB,GAAG;IAC3B,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;CACP,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,IAAU,EAAE,EAAE;IACjC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO;IACR,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAChC,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAChC,OAAO;IACR,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IAC3E,IAAI,QAAQ,EAAE,CAAC;QACd,OAAO,QAAkB,CAAC;IAC3B,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,IAAU,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEnF,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC7C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,8BAA8B;IAC9B,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yCAAyC;IACzC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC7E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QAChG,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,IAAc,CAAC,EAAE,CAAC;QAC3F,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,6BAA6B;IAC7B,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3E,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAEnC,IAAI,OAAO,CAAC,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,6BAA6B;IAC7B,IAAI,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzF,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE;IACxC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAE7B,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACd,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;IACpE,MAAM,YAAY,GAAG,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;IAEpE,OAAO,WAAW,KAAK,YAAY,IAAI,WAAW,CAAC;AACpD,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,IAAU,EAAE,EAAE;IACxC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACtB,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAQ,EAAE;IAC1D,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;IAE1C,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAA2B,CAAC;IAC5D,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,EAAE;QAChD,IAAI,CAAC,cAAc,CAAC,IAAY,CAAC,EAAE,CAAC;YACnC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,sBAAsB,CAAC,IAAY,CAAC,EAAE,CAAC;YAC3C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,WAAW,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACvC,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAC3C,IAAI,aAAa,GAAG,IAAI,IAAI,mBAAmB,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAwC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACtH,IAAI,QAAQ,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QACrC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,QAAQ,GAAS,WAAW,CAAC;IACjC,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,CAAC;QACnF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAEpC,IAAI,QAAQ,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC1B,SAAS;QACV,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAElE,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QACtD,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;YACvB,SAAS;QACV,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;YACrC,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM;QACP,CAAC;IACF,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3C,IAAI,CAAC,iBAAiB,CAAC,IAAY,CAAC,EAAE,CAAC;YACtC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,qBAAqB,CAAC,IAAY,CAAC,EAAE,CAAC;YAC1C,OAAO,KAAK,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC,CAAS,CAAC;IAEX,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}
+{"version":3,"file":"readability.js","sourceRoot":"","sources":["../../../src/extract-hast/readability.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE9E,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;AAE5G,MAAM,OAAO,GAAG;IACf,MAAM,EAAE,kCAAkC;IAC1C,MAAM,EAAE,4CAA4C;IACpD,yBAAyB,EAAE,8BAA8B;IACzD,kBAAkB,EACjB,yQAAyQ;IAC1Q,oBAAoB,EAAE,mDAAmD;CACzE,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;AAE/G,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEpF,MAAM,mBAAmB,GAAG;IAC3B,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;CACP,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,IAAU,EAAE,EAAE;IACjC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO;IACR,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAChC,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAChC,OAAO;IACR,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IAC3E,IAAI,QAAQ,EAAE,CAAC;QACd,OAAO,QAAkB,CAAC;IAC3B,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,IAAU,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEnF,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC7C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,8BAA8B;IAC9B,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yCAAyC;IACzC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC7E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QAChG,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,IAAc,CAAC,EAAE,CAAC;QAC3F,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,6BAA6B;IAC7B,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3E,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAEnC,IAAI,OAAO,CAAC,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,6BAA6B;IAC7B,IAAI,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzF,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE;IACxC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAE7B,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACd,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;IACpE,MAAM,YAAY,GAAG,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;IAEpE,OAAO,WAAW,KAAK,YAAY,IAAI,WAAW,CAAC;AACpD,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,IAAU,EAAE,EAAE;IACxC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACtB,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAQ,EAAE;IAC1D,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;IAC1C,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAEpC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;QACzC,IAAI,CAAC,cAAc,CAAC,IAAY,CAAC,EAAE,CAAC;YACnC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,sBAAsB,CAAC,IAAY,CAAC,EAAE,CAAC;YAC3C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,WAAW,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACvC,CAAC;IAED,MAAM,eAAe,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAClD,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,GACvB,eAAe,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,MAAM,GAAG,IAAI;QAC5E,CAAC,CAAE,CAAC,WAAmB,EAAE,eAAe,CAAW;QACnD,CAAC,CAAE,CAAC,IAAY,EAAE,QAAQ,CAAW,CAAC;IAExC,IAAI,aAAa,GAAG,IAAI,IAAI,mBAAmB,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAwC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACtH,IAAI,QAAQ,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QACrC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,CAAC;QACtF,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QAE1C,IAAI,WAAW,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC7B,SAAS;QACV,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAElE,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QACtD,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;YACvB,SAAS;QACV,CAAC;QAED,IAAI,WAAW,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;YACxC,QAAQ,GAAG,OAAO,CAAC;YACnB,QAAQ,GAAG,WAAW,CAAC;YACvB,MAAM;QACP,CAAC;IACF,CAAC;IAED,MAAM,iBAAiB,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE;QACnD,IAAI,CAAC,iBAAiB,CAAC,IAAY,CAAC,EAAE,CAAC;YACtC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,qBAAqB,CAAC,IAAY,CAAC,EAAE,CAAC;YAC1C,OAAO,KAAK,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC,CAAS,CAAC;IAEX,MAAM,SAAS,GAAG,YAAY,CAAC,iBAAiB,CAAC,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC9G,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}

package/dist/types/loaders/puppeteer.d.ts

@@ -0,0 +1,2 @@
+import type { PuppeteerLaunchOptions } from "puppeteer";
+export declare const loadHtml: (url: string, ctx?: PuppeteerLaunchOptions) => Promise<string>;

package/dist/types/loaders/puppeteer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"puppeteer.js","sourceRoot":"","sources":["../../../src/loaders/puppeteer.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,WAAW,CAAC;AAGlC,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAE,GAA4B,EAAE,EAAE;IAC3E,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,MAAM,CACrC,GAAG,IAAI;QACN,QAAQ,EAAE,IAAI;QACd,IAAI,EAAE,CAAC,cAAc,EAAE,0BAA0B,CAAC;KAClD,CACD,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAErB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE9D,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IAEnB,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webforai",
-  "version": "1.3.3",
+  "version": "1.4.1",
   "description": "A library that provides a web interface for AI",
   "author": "inaridiy",
   "license": "MIT",
@@ -51,6 +51,11 @@
       "types": "./dist/types/loaders/cf-puppeteer.d.ts",
       "import": "./dist/loaders/cf-puppeteer.js",
       "require": "./dist/cjs/loaders/cf-puppeteer.js"
+    },
+    "./loaders/puppeteer": {
+      "types": "./dist/types/loaders/puppeteer.d.ts",
+      "import": "./dist/loaders/puppeteer.js",
+      "require": "./dist/cjs/loaders/puppeteer.js"
     }
   },
   "typesVersions": {
@@ -63,12 +68,19 @@
       ],
       "loaders/cf-puppeteer": [
         "./dist/types/loaders/cf-puppeteer.d.ts"
+      ],
+      "loaders/fetch": [
+        "./dist/types/loaders/fetch.d.ts"
+      ],
+      "loaders/puppeteer": [
+        "./dist/types/loaders/puppeteer.d.ts"
       ]
     }
   },
   "peerDependencies": {
     "@cloudflare/puppeteer": "^0.0.6",
-    "playwright": "^1.0.0"
+    "playwright": "^1.0.0",
+    "puppeteer": "^22.10.0"
   },
   "peerDependenciesMeta": {
     "playwright": {
@@ -76,6 +88,9 @@
     },
     "@cloudflare/puppeteer": {
       "optional": true
+    },
+    "puppeteer": {
+      "optional": true
     }
   },
   "dependencies": {
@@ -90,6 +105,7 @@
     "mdast-util-gfm": "^3.0.0",
     "mdast-util-math": "^3.0.0",
     "mdast-util-to-markdown": "^2.1.0",
+    "puppeteer": "^22.10.0",
     "trim-trailing-lines": "^2.1.0",
     "unist-util-filter": "^5.0.1",
     "unist-util-parents": "^3.0.0",