webforai 1.3.1 → 1.3.3

package/dist/cjs/extract-hast/readability.js

@@ -30,6 +30,7 @@ const UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert
 const REGEXPS = {
   hidden: /hidden|invisible|fallback-image/i,
   byline: /byline|author|dateline|writtenby|p-author/i,
+  specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
   unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore/i,
   okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
 };
@@ -89,18 +90,31 @@ const unlikelyElementFilter = (node) => {
     return true;
   }
   const element = node;
-  const match = (0, import_utils.matchString)(element);
   if (["body", "article", "main", "section", "a"].includes(element.tagName)) {
     return true;
   }
   if ((0, import_utils.hasAncestors)(element, ["table", "code"], 3)) {
     return true;
   }
+  const match = (0, import_utils.matchString)(element);
+  if (REGEXPS.specialUnlikelyCandidates.test(match)) {
+    return false;
+  }
   if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
     return false;
   }
   return true;
 };
+const isImageLink = (element) => {
+  const a = (0, import_hast_util_select.select)("a", element);
+  const img = (0, import_hast_util_select.select)("img", a);
+  if (!(a && img)) {
+    return false;
+  }
+  const imgFilename = img.properties.src?.toString().split("/").pop();
+  const hrefFilename = a.properties.href?.toString().split("/").pop();
+  return imgFilename === hrefFilename && imgFilename;
+};
 const removeEmptyFilter = (node) => {
   if (node.type !== "element") {
     return true;
@@ -109,6 +123,9 @@ const removeEmptyFilter = (node) => {
   if (!PARAGRAPH_TAGS.includes(element.tagName)) {
     return true;
   }
+  if (isImageLink(element)) {
+    return true;
+  }
   const text = (0, import_hast_util_to_string.toString)(element);
   if (text.length < 10) {
     return false;
@@ -119,7 +136,7 @@ const readabilityExtractHast = (hast) => {
   const lang = String((0, import_hast_util_select.select)("html", hast)?.properties.lang || tryGetLang(hast) || "en");
   const body = (0, import_hast_util_select.select)("body", hast) ?? hast;
   const proxiedHast = (0, import_unist_util_parents.parents)(body);
-  let baseFilterd = (0, import_unist_util_filter.filter)(proxiedHast, (node) => {
+  const baseFilterd = (0, import_unist_util_filter.filter)(proxiedHast, (node) => {
     if (!metadataFilter(node)) {
       return false;
     }
@@ -133,17 +150,7 @@ const readabilityExtractHast = (hast) => {
   }
   const baseText = (0, import_hast_util_to_string.toString)(baseFilterd);
   let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
-  if (baseText.length > minimalLength) {
-    const filterd = (0, import_unist_util_filter.filter)(baseFilterd, (node) => {
-      if (!unlikelyElementFilter(node)) {
-        return false;
-      }
-      return true;
-    });
-    if (filterd) {
-      baseFilterd = filterd;
-    }
-  } else {
+  if (baseText.length < minimalLength) {
     minimalLength = Math.max(0, baseText.length - 200);
   }
   let bodyTree = baseFilterd;
@@ -168,6 +175,9 @@ const readabilityExtractHast = (hast) => {
     if (!removeEmptyFilter(node)) {
       return false;
     }
+    if (!unlikelyElementFilter(node)) {
+      return false;
+    }
     return true;
   });
   return finalTree;

package/dist/cjs/extract-hast/utils.js

@@ -24,7 +24,7 @@ __export(utils_exports, {
   matchString: () => matchString
 });
 module.exports = __toCommonJS(utils_exports);
-const matchString = (element) => `${element.tagName} ${element.properties.id} ${classnames(element).join(" ")} }`;
+const matchString = (element) => `${element.tagName} ${element.properties.id} ${classnames(element).join(" ")}`;
 const classnames = (element) => {
   if (Array.isArray(element.properties.className)) {
     return element.properties.className;

package/dist/cjs/extract-mdast.js

@@ -0,0 +1,47 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var extract_mdast_exports = {};
+__export(extract_mdast_exports, {
+  extractMdast: () => extractMdast
+});
+module.exports = __toCommonJS(extract_mdast_exports);
+var import_unist_util_filter = require("unist-util-filter");
+const DECLATION_TYPES = ["blockquote", "strong", "emphasis", "delete"];
+const emptyDeclarationFilter = (node) => {
+  if (!DECLATION_TYPES.includes(node.type)) {
+    return true;
+  }
+  if (node.children.length === 0) {
+    return false;
+  }
+  return true;
+};
+const extractMdast = (node) => {
+  const extracted = (0, import_unist_util_filter.filter)(node, (node2) => {
+    if (!emptyDeclarationFilter(node2)) {
+      return false;
+    }
+    return true;
+  });
+  return extracted;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  extractMdast
+});

package/dist/cjs/html-to-markdown.test.js

@@ -1,10 +1,35 @@
 "use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
 var import_fastest_levenshtein = require("fastest-levenshtein");
 var import_vitest = require("vitest");
 var import_html_to_markdown = require("./html-to-markdown");
+var import_fetch = require("./loaders/fetch");
 const html = `
 <h1>Hello, world!</h1>
 <p>This is a paragraph.</p>
+<a href="/example.html">Example</a>
+<img src="/example.jpg" alt="Example" />
 <ul>
   <li>Item 1</li>
   <li>Item 2</li>
@@ -14,13 +39,108 @@ const expected = `# Hello, world!
 
 This is a paragraph.
 
+[Example](/example.html)
+
+![Example](/example.jpg)
+
 * Item 1
 * Item 2
 `;
+const baseUrlReplaced = `# Hello, world!
+
+This is a paragraph.
+
+[Example](https://example.com/example.html)
+ 
+![Example](https://example.com/example.jpg)
+
+* Item 1
+* Item 2
+`;
+const linkAsText = `# Hello, world!
+
+This is a paragraph.
+
+Example
+
+![Example](/example.jpg)
+
+- Item 1
+- Item 2
+`;
+const imageHidden = `# Hello, world!
+
+This is a paragraph.
+
+[Example](/example.html)
+
+- Item 1
+- Item 2
+`;
+const htmlTable = `
+<table>
+  <tr>
+    <th>Header 1</th>
+    <th>Header 2</th>
+  </tr>
+  <tr>
+    <td>Cell 1</td>
+    <td>Cell 2</td>
+  </tr>
+</table>
+`;
+const expectedTableMarkdown = `
+| Header 1 | Header 2 |
+| -------- | -------- |
+| Cell 1   | Cell 2   |
+`;
+const expectedTableText = `Header 1  Header 2
+Cell 1    Cell 2`;
 (0, import_vitest.describe)("htmlToMarkdown", () => {
   (0, import_vitest.it)("should convert HTML to Markdown", () => {
     const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html, { extractors: false });
     const d = (0, import_fastest_levenshtein.distance)(markdown, expected);
-    (0, import_vitest.expect)(d).lte(2);
+    (0, import_vitest.expect)(d).lte(5);
+  });
+  (0, import_vitest.it)("should convert HTML to Markdown with replaced base URL", () => {
+    const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html, { baseUrl: "https://example.com", extractors: false });
+    const d = (0, import_fastest_levenshtein.distance)(markdown, baseUrlReplaced);
+    (0, import_vitest.expect)(d).lte(5);
+  });
+  (0, import_vitest.it)("should convert HTML to Markdown with links as text", () => {
+    const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html, { linkAsText: true, extractors: false });
+    const d = (0, import_fastest_levenshtein.distance)(markdown, linkAsText);
+    (0, import_vitest.expect)(d).lte(5);
+  });
+  (0, import_vitest.it)("should convert HTML to Markdown with hidden images", () => {
+    const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html, { hideImage: true, extractors: false });
+    const d = (0, import_fastest_levenshtein.distance)(markdown, imageHidden);
+    (0, import_vitest.expect)(d).lte(5);
+  });
+  (0, import_vitest.it)("should convert HTML table to Markdown table", () => {
+    const markdown = (0, import_html_to_markdown.htmlToMarkdown)(htmlTable, { extractors: false });
+    const d = (0, import_fastest_levenshtein.distance)(markdown, expectedTableMarkdown);
+    (0, import_vitest.expect)(d).lte(5);
+  });
+  (0, import_vitest.it)("should convert HTML table with table as text option", () => {
+    const markdown = (0, import_html_to_markdown.htmlToMarkdown)(htmlTable, { tableAsText: true, extractors: false });
+    const d = (0, import_fastest_levenshtein.distance)(markdown, expectedTableText);
+    (0, import_vitest.expect)(d).lte(10);
+  });
+});
+(0, import_vitest.describe)("htmlToMarkdown E2E", () => {
+  (0, import_vitest.it)("should convert npm README to Markdown ", async () => {
+    const html2 = await (0, import_fetch.loadHtml)("https://www.npmjs.com/package/webforai");
+    const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html2, { baseUrl: "https://www.npmjs.com/package/webforai" });
+    const original = await import("../README.md?raw");
+    const d = (0, import_fastest_levenshtein.distance)(markdown, original.default);
+    (0, import_vitest.expect)(d).lte(400);
+  });
+  (0, import_vitest.it)("should convert GitHub README to Markdown ", async () => {
+    const html2 = await (0, import_fetch.loadHtml)("https://github.com/inaridiy/webforai");
+    const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html2, { baseUrl: "https://www.npmjs.com/package/webforai" });
+    const original = await import("../../../README.md?raw");
+    const d = (0, import_fastest_levenshtein.distance)(markdown, original.default);
+    (0, import_vitest.expect)(d).lte(200);
   });
 });

package/dist/cjs/html-to-mdast.js

@@ -24,6 +24,7 @@ module.exports = __toCommonJS(html_to_mdast_exports);
 var import_hast_util_from_html = require("hast-util-from-html");
 var import_hast_util_to_mdast = require("hast-util-to-mdast");
 var import_extract_hast = require("./extract-hast");
+var import_extract_mdast = require("./extract-mdast");
 var import_custom_a_handler = require("./mdast-handlers/custom-a-handler");
 var import_custom_code_handler = require("./mdast-handlers/custom-code-handler");
 var import_custom_div_handler = require("./mdast-handlers/custom-div-handler");
@@ -46,7 +47,8 @@ const htmlToMdast = (htmlOrHast, options) => {
       br: import_empty_handler.emptyHandler
     }
   });
-  return mdast;
+  const extractedMdast = (0, import_extract_mdast.extractMdast)(mdast);
+  return extractedMdast;
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/link-replacer.test.js

@@ -0,0 +1,47 @@
+"use strict";
+var import_vitest = require("vitest");
+var import_link_replacer = require("./link-replacer");
+const markdown = `# Hello, world!
+
+This is a paragraph.
+
+[Example](/example.html)
+
+![Example](/example.jpg)
+
+[Absolute Link](https://www.google.com)
+
+[Link with hash](/page#hash)
+
+[Link with query](/page?query=string)`;
+const expected = `# Hello, world!
+
+This is a paragraph.
+
+[Example](https://example.com/example.html)
+
+![Example](https://example.com/example.jpg)
+
+[Absolute Link](https://www.google.com)
+
+[Link with hash](https://example.com/page#hash)
+
+[Link with query](https://example.com/page?query=string)`;
+(0, import_vitest.describe)("linkReplacer", () => {
+  (0, import_vitest.it)("should replace relative links", () => {
+    const replaced = (0, import_link_replacer.linkReplacer)(markdown, "https://example.com");
+    (0, import_vitest.expect)(replaced).toEqual(expected);
+  });
+  (0, import_vitest.it)("should not replace absolute links", () => {
+    const replaced = (0, import_link_replacer.linkReplacer)("[Absolute Link](https://www.google.com)", "https://example.com");
+    (0, import_vitest.expect)(replaced).toEqual("[Absolute Link](https://www.google.com)");
+  });
+  (0, import_vitest.it)("should handle links with hashes", () => {
+    const replaced = (0, import_link_replacer.linkReplacer)("[Link with hash](/page#hash)", "https://example.com");
+    (0, import_vitest.expect)(replaced).toEqual("[Link with hash](https://example.com/page#hash)");
+  });
+  (0, import_vitest.it)("should handle links with query parameters", () => {
+    const replaced = (0, import_link_replacer.linkReplacer)("[Link with query](/page?query=string)", "https://example.com");
+    (0, import_vitest.expect)(replaced).toEqual("[Link with query](https://example.com/page?query=string)");
+  });
+});

package/dist/cjs/loaders/fetch.js

@@ -0,0 +1,34 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var fetch_exports = {};
+__export(fetch_exports, {
+  USER_AGENT: () => USER_AGENT,
+  loadHtml: () => loadHtml
+});
+module.exports = __toCommonJS(fetch_exports);
+const USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
+const loadHtml = async (url, userAgent = USER_AGENT) => {
+  const response = await fetch(url, { headers: { "User-Agent": userAgent } });
+  return response.text();
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  USER_AGENT,
+  loadHtml
+});

package/dist/cjs/loaders/fetch.test.js

@@ -0,0 +1,9 @@
+"use strict";
+var import_vitest = require("vitest");
+var import_fetch = require("./fetch");
+(0, import_vitest.describe)("Fetch loader", () => {
+  (0, import_vitest.it)("should load the HTML of a URL", async () => {
+    const html = await (0, import_fetch.loadHtml)("https://example.com");
+    (0, import_vitest.expect)(html).toContain("Example Domain");
+  });
+});

package/dist/cjs/loaders/playwright.js

@@ -22,23 +22,14 @@ __export(playwright_exports, {
 });
 module.exports = __toCommonJS(playwright_exports);
 var import_playwright = require("playwright");
-const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const loadHtml = async (url, context) => {
-  let _browser = null;
-  let _context;
-  if (context) {
-    _context = context;
-  } else {
-    _browser = await import_playwright.chromium.launch({ headless: true });
-    _context = await _browser.newContext();
-  }
+  const _context = context ?? await import_playwright.chromium.launch({ headless: true });
   const page = await _context.newPage();
-  await page.goto(url);
-  await Promise.race([page.waitForLoadState("networkidle"), sleep(1e4)]);
+  await page.goto(url, { waitUntil: "load" });
   const html = await page.content();
   await page.close();
-  if (_browser) {
-    await _browser.close();
+  if (!context) {
+    await _context.close();
   }
   return html;
 };

package/dist/cjs/loaders/playwright.test.js

@@ -0,0 +1,17 @@
+"use strict";
+var import_playwright = require("playwright");
+var import_vitest = require("vitest");
+var import_playwright2 = require("./playwright");
+(0, import_vitest.describe)("Playwright loader", () => {
+  (0, import_vitest.it)("should load the HTML of a URL", async () => {
+    const html = await (0, import_playwright2.loadHtml)("https://example.com");
+    (0, import_vitest.expect)(html).toContain("Example Domain");
+  });
+  (0, import_vitest.it)("should load the HTML of a URL using a custom context", async () => {
+    const context = await import_playwright.chromium.launch({ headless: true });
+    const html = await (0, import_playwright2.loadHtml)("https://example.com", context);
+    (0, import_vitest.expect)(html).toContain("Example Domain");
+    (0, import_vitest.expect)(context.isConnected()).toBe(true);
+    await context.close();
+  });
+});

package/dist/cjs/mdast-handlers/custom-a-handler.js

@@ -25,7 +25,11 @@ var import_hast_util_to_mdast = require("hast-util-to-mdast");
 var import_hast_util_to_string = require("hast-util-to-string");
 const customAHandler = (options) => (state, node) => {
   if (options?.asText) {
-    const link2 = { type: "text", value: (0, import_hast_util_to_string.toString)(node) };
+    const text = (0, import_hast_util_to_string.toString)(node);
+    if (3 >= text.length) {
+      return void 0;
+    }
+    const link2 = { type: "text", value: text };
     state.patch(node, link2);
     return link2;
   }

package/dist/cjs/mdast-to-markdown.js

@@ -28,7 +28,8 @@ var import_mdast_util_to_markdown = require("mdast-util-to-markdown");
 var import_link_replacer = require("./link-replacer");
 var import_mdast_utils = require("./utils/mdast-utils");
 const DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
-  extensions: [(0, import_mdast_util_gfm.gfmToMarkdown)(), (0, import_mdast_util_math.mathToMarkdown)()]
+  extensions: [(0, import_mdast_util_gfm.gfmToMarkdown)(), (0, import_mdast_util_math.mathToMarkdown)()],
+  bullet: "-"
 };
 const mdastToMarkdown = (mdast, options) => {
   const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };

package/dist/extract-hast/readability.js

@@ -8,6 +8,7 @@ var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert",
 var REGEXPS = {
   hidden: /hidden|invisible|fallback-image/i,
   byline: /byline|author|dateline|writtenby|p-author/i,
+  specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
   unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore/i,
   okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
 };
@@ -67,18 +68,31 @@ var unlikelyElementFilter = (node) => {
     return true;
   }
   const element = node;
-  const match = matchString(element);
   if (["body", "article", "main", "section", "a"].includes(element.tagName)) {
     return true;
   }
   if (hasAncestors(element, ["table", "code"], 3)) {
     return true;
   }
+  const match = matchString(element);
+  if (REGEXPS.specialUnlikelyCandidates.test(match)) {
+    return false;
+  }
   if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
     return false;
   }
   return true;
 };
+var isImageLink = (element) => {
+  const a = select("a", element);
+  const img = select("img", a);
+  if (!(a && img)) {
+    return false;
+  }
+  const imgFilename = img.properties.src?.toString().split("/").pop();
+  const hrefFilename = a.properties.href?.toString().split("/").pop();
+  return imgFilename === hrefFilename && imgFilename;
+};
 var removeEmptyFilter = (node) => {
   if (node.type !== "element") {
     return true;
@@ -87,6 +101,9 @@ var removeEmptyFilter = (node) => {
   if (!PARAGRAPH_TAGS.includes(element.tagName)) {
     return true;
   }
+  if (isImageLink(element)) {
+    return true;
+  }
   const text = hastToString(element);
   if (text.length < 10) {
     return false;
@@ -97,7 +114,7 @@ var readabilityExtractHast = (hast) => {
   const lang = String(select("html", hast)?.properties.lang || tryGetLang(hast) || "en");
   const body = select("body", hast) ?? hast;
   const proxiedHast = parents(body);
-  let baseFilterd = filter(proxiedHast, (node) => {
+  const baseFilterd = filter(proxiedHast, (node) => {
     if (!metadataFilter(node)) {
       return false;
     }
@@ -111,17 +128,7 @@ var readabilityExtractHast = (hast) => {
   }
   const baseText = hastToString(baseFilterd);
   let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
-  if (baseText.length > minimalLength) {
-    const filterd = filter(baseFilterd, (node) => {
-      if (!unlikelyElementFilter(node)) {
-        return false;
-      }
-      return true;
-    });
-    if (filterd) {
-      baseFilterd = filterd;
-    }
-  } else {
+  if (baseText.length < minimalLength) {
     minimalLength = Math.max(0, baseText.length - 200);
   }
   let bodyTree = baseFilterd;
@@ -146,6 +153,9 @@ var readabilityExtractHast = (hast) => {
     if (!removeEmptyFilter(node)) {
       return false;
     }
+    if (!unlikelyElementFilter(node)) {
+      return false;
+    }
     return true;
   });
   return finalTree;

package/dist/extract-hast/utils.js

@@ -1,5 +1,5 @@
 // src/extract-hast/utils.ts
-var matchString = (element) => `${element.tagName} ${element.properties.id} ${classnames(element).join(" ")} }`;
+var matchString = (element) => `${element.tagName} ${element.properties.id} ${classnames(element).join(" ")}`;
 var classnames = (element) => {
   if (Array.isArray(element.properties.className)) {
     return element.properties.className;

package/dist/extract-mdast.js

@@ -0,0 +1,24 @@
+// src/extract-mdast.ts
+import { filter } from "unist-util-filter";
+var DECLATION_TYPES = ["blockquote", "strong", "emphasis", "delete"];
+var emptyDeclarationFilter = (node) => {
+  if (!DECLATION_TYPES.includes(node.type)) {
+    return true;
+  }
+  if (node.children.length === 0) {
+    return false;
+  }
+  return true;
+};
+var extractMdast = (node) => {
+  const extracted = filter(node, (node2) => {
+    if (!emptyDeclarationFilter(node2)) {
+      return false;
+    }
+    return true;
+  });
+  return extracted;
+};
+export {
+  extractMdast
+};

package/dist/html-to-markdown.test.js

@@ -2,9 +2,12 @@
 import { distance } from "fastest-levenshtein";
 import { describe, expect, it } from "vitest";
 import { htmlToMarkdown } from "./html-to-markdown.js";
+import { loadHtml } from "./loaders/fetch.js";
 var html = `
 <h1>Hello, world!</h1>
 <p>This is a paragraph.</p>
+<a href="/example.html">Example</a>
+<img src="/example.jpg" alt="Example" />
 <ul>
   <li>Item 1</li>
   <li>Item 2</li>
@@ -14,13 +17,108 @@ var expected = `# Hello, world!
 
 This is a paragraph.
 
+[Example](/example.html)
+
+![Example](/example.jpg)
+
 * Item 1
 * Item 2
 `;
+var baseUrlReplaced = `# Hello, world!
+
+This is a paragraph.
+
+[Example](https://example.com/example.html)
+ 
+![Example](https://example.com/example.jpg)
+
+* Item 1
+* Item 2
+`;
+var linkAsText = `# Hello, world!
+
+This is a paragraph.
+
+Example
+
+![Example](/example.jpg)
+
+- Item 1
+- Item 2
+`;
+var imageHidden = `# Hello, world!
+
+This is a paragraph.
+
+[Example](/example.html)
+
+- Item 1
+- Item 2
+`;
+var htmlTable = `
+<table>
+  <tr>
+    <th>Header 1</th>
+    <th>Header 2</th>
+  </tr>
+  <tr>
+    <td>Cell 1</td>
+    <td>Cell 2</td>
+  </tr>
+</table>
+`;
+var expectedTableMarkdown = `
+| Header 1 | Header 2 |
+| -------- | -------- |
+| Cell 1   | Cell 2   |
+`;
+var expectedTableText = `Header 1  Header 2
+Cell 1    Cell 2`;
 describe("htmlToMarkdown", () => {
   it("should convert HTML to Markdown", () => {
     const markdown = htmlToMarkdown(html, { extractors: false });
     const d = distance(markdown, expected);
-    expect(d).lte(2);
+    expect(d).lte(5);
+  });
+  it("should convert HTML to Markdown with replaced base URL", () => {
+    const markdown = htmlToMarkdown(html, { baseUrl: "https://example.com", extractors: false });
+    const d = distance(markdown, baseUrlReplaced);
+    expect(d).lte(5);
+  });
+  it("should convert HTML to Markdown with links as text", () => {
+    const markdown = htmlToMarkdown(html, { linkAsText: true, extractors: false });
+    const d = distance(markdown, linkAsText);
+    expect(d).lte(5);
+  });
+  it("should convert HTML to Markdown with hidden images", () => {
+    const markdown = htmlToMarkdown(html, { hideImage: true, extractors: false });
+    const d = distance(markdown, imageHidden);
+    expect(d).lte(5);
+  });
+  it("should convert HTML table to Markdown table", () => {
+    const markdown = htmlToMarkdown(htmlTable, { extractors: false });
+    const d = distance(markdown, expectedTableMarkdown);
+    expect(d).lte(5);
+  });
+  it("should convert HTML table with table as text option", () => {
+    const markdown = htmlToMarkdown(htmlTable, { tableAsText: true, extractors: false });
+    const d = distance(markdown, expectedTableText);
+    expect(d).lte(10);
+  });
+});
+describe("htmlToMarkdown E2E", () => {
+  it("should convert npm README to Markdown ", async () => {
+    const html2 = await loadHtml("https://www.npmjs.com/package/webforai");
+    const markdown = htmlToMarkdown(html2, { baseUrl: "https://www.npmjs.com/package/webforai" });
+    const original = await import("../README.md?raw");
+    const d = distance(markdown, original.default);
+    expect(d).lte(400);
+  });
+  it("should convert GitHub README to Markdown ", async () => {
+    const html2 = await loadHtml("https://github.com/inaridiy/webforai");
+    const markdown = htmlToMarkdown(html2, { baseUrl: "https://www.npmjs.com/package/webforai" });
+    const original = await import("../../../README.md?raw");
+    const d = distance(markdown, original.default);
+    expect(d).lte(200);
   });
 });

package/dist/html-to-mdast.js

@@ -2,6 +2,7 @@
 import { fromHtml } from "hast-util-from-html";
 import { toMdast } from "hast-util-to-mdast";
 import { extractHast } from "./extract-hast/index.js";
+import { extractMdast } from "./extract-mdast.js";
 import { customAHandler } from "./mdast-handlers/custom-a-handler.js";
 import { customCodeHandler } from "./mdast-handlers/custom-code-handler.js";
 import { customDivHandler } from "./mdast-handlers/custom-div-handler.js";
@@ -24,7 +25,8 @@ var htmlToMdast = (htmlOrHast, options) => {
       br: emptyHandler
     }
   });
-  return mdast;
+  const extractedMdast = extractMdast(mdast);
+  return extractedMdast;
 };
 export {
   htmlToMdast

package/dist/link-replacer.test.js

@@ -0,0 +1,47 @@
+// src/link-replacer.test.ts
+import { describe, expect, it } from "vitest";
+import { linkReplacer } from "./link-replacer.js";
+var markdown = `# Hello, world!
+
+This is a paragraph.
+
+[Example](/example.html)
+
+![Example](/example.jpg)
+
+[Absolute Link](https://www.google.com)
+
+[Link with hash](/page#hash)
+
+[Link with query](/page?query=string)`;
+var expected = `# Hello, world!
+
+This is a paragraph.
+
+[Example](https://example.com/example.html)
+
+![Example](https://example.com/example.jpg)
+
+[Absolute Link](https://www.google.com)
+
+[Link with hash](https://example.com/page#hash)
+
+[Link with query](https://example.com/page?query=string)`;
+describe("linkReplacer", () => {
+  it("should replace relative links", () => {
+    const replaced = linkReplacer(markdown, "https://example.com");
+    expect(replaced).toEqual(expected);
+  });
+  it("should not replace absolute links", () => {
+    const replaced = linkReplacer("[Absolute Link](https://www.google.com)", "https://example.com");
+    expect(replaced).toEqual("[Absolute Link](https://www.google.com)");
+  });
+  it("should handle links with hashes", () => {
+    const replaced = linkReplacer("[Link with hash](/page#hash)", "https://example.com");
+    expect(replaced).toEqual("[Link with hash](https://example.com/page#hash)");
+  });
+  it("should handle links with query parameters", () => {
+    const replaced = linkReplacer("[Link with query](/page?query=string)", "https://example.com");
+    expect(replaced).toEqual("[Link with query](https://example.com/page?query=string)");
+  });
+});

package/dist/loaders/fetch.js

@@ -0,0 +1,10 @@
+// src/loaders/fetch.ts
+var USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
+var loadHtml = async (url, userAgent = USER_AGENT) => {
+  const response = await fetch(url, { headers: { "User-Agent": userAgent } });
+  return response.text();
+};
+export {
+  USER_AGENT,
+  loadHtml
+};

package/dist/loaders/fetch.test.js

@@ -0,0 +1,9 @@
+// src/loaders/fetch.test.ts
+import { describe, expect, it } from "vitest";
+import { loadHtml } from "./fetch.js";
+describe("Fetch loader", () => {
+  it("should load the HTML of a URL", async () => {
+    const html = await loadHtml("https://example.com");
+    expect(html).toContain("Example Domain");
+  });
+});

package/dist/loaders/playwright.js

@@ -1,22 +1,13 @@
 // src/loaders/playwright.ts
 import { chromium } from "playwright";
-var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 var loadHtml = async (url, context) => {
-  let _browser = null;
-  let _context;
-  if (context) {
-    _context = context;
-  } else {
-    _browser = await chromium.launch({ headless: true });
-    _context = await _browser.newContext();
-  }
+  const _context = context ?? await chromium.launch({ headless: true });
   const page = await _context.newPage();
-  await page.goto(url);
-  await Promise.race([page.waitForLoadState("networkidle"), sleep(1e4)]);
+  await page.goto(url, { waitUntil: "load" });
   const html = await page.content();
   await page.close();
-  if (_browser) {
-    await _browser.close();
+  if (!context) {
+    await _context.close();
   }
   return html;
 };

package/dist/loaders/playwright.test.js

@@ -0,0 +1,17 @@
+// src/loaders/playwright.test.ts
+import { chromium } from "playwright";
+import { describe, expect, it } from "vitest";
+import { loadHtml } from "./playwright.js";
+describe("Playwright loader", () => {
+  it("should load the HTML of a URL", async () => {
+    const html = await loadHtml("https://example.com");
+    expect(html).toContain("Example Domain");
+  });
+  it("should load the HTML of a URL using a custom context", async () => {
+    const context = await chromium.launch({ headless: true });
+    const html = await loadHtml("https://example.com", context);
+    expect(html).toContain("Example Domain");
+    expect(context.isConnected()).toBe(true);
+    await context.close();
+  });
+});

package/dist/mdast-handlers/custom-a-handler.js

@@ -3,7 +3,11 @@ import { defaultHandlers } from "hast-util-to-mdast";
 import { toString as hastToString } from "hast-util-to-string";
 var customAHandler = (options) => (state, node) => {
   if (options?.asText) {
-    const link2 = { type: "text", value: hastToString(node) };
+    const text = hastToString(node);
+    if (3 >= text.length) {
+      return void 0;
+    }
+    const link2 = { type: "text", value: text };
     state.patch(node, link2);
     return link2;
   }

package/dist/mdast-to-markdown.js

@@ -5,7 +5,8 @@ import { toMarkdown } from "mdast-util-to-markdown";
 import { linkReplacer } from "./link-replacer.js";
 import { warpRoot } from "./utils/mdast-utils.js";
 var DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
-  extensions: [gfmToMarkdown(), mathToMarkdown()]
+  extensions: [gfmToMarkdown(), mathToMarkdown()],
+  bullet: "-"
 };
 var mdastToMarkdown = (mdast, options) => {
   const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };

package/dist/types/extract-hast/readability.js.map

@@ -1 +1 @@
-{"version":3,"file":"readability.js","sourceRoot":"","sources":["../../../src/extract-hast/readability.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAU9E,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;AAE5G,MAAM,OAAO,GAAG;IACf,MAAM,EAAE,kCAAkC;IAC1C,MAAM,EAAE,4CAA4C;IACpD,kBAAkB,EACjB,yQAAyQ;IAC1Q,oBAAoB,EAAE,mDAAmD;CACzE,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;AAE/G,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEpF,MAAM,mBAAmB,GAAG;IAC3B,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;CACP,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,IAAU,EAAE,EAAE;IACjC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO;IACR,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAChC,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAChC,OAAO;IACR,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IAC3E,IAAI,QAAQ,EAAE,CAAC;QACd,OAAO,QAAkB,CAAC;IAC3B,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,IAAU,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEnF,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC7C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,8BAA8B;IAC9B,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yCAAyC;IACzC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC7E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QAChG,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,IAAc,CAAC,EAAE,CAAC;QAC3F,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAChC,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAEnC,6BAA6B;IAC7B,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3E,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,6BAA6B;IAC7B,IAAI,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzF,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,IAAU,EAAE,EAAE;IACxC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAChC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACtB,OAAO,KAAK,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAQ,EAAE;IAC1D,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;IAE1C,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAA2B,CAAC;IAC5D,IAAI,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,EAAE;QAC9C,IAAI,CAAC,cAAc,CAAC,IAAY,CAAC,EAAE,CAAC;YACnC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,sBAAsB,CAAC,IAAY,CAAC,EAAE,CAAC;YAC3C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,WAAW,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACvC,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAC3C,IAAI,aAAa,GAAG,IAAI,IAAI,mBAAmB,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAwC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACtH,IAAI,QAAQ,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,EAAE;YAC5C,IAAI,CAAC,qBAAqB,CAAC,IAAY,CAAC,EAAE,CAAC;gBAC1C,OAAO,KAAK,CAAC;YACd,CAAC;YACD,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QACH,IAAI,OAAO,EAAE,CAAC;YACb,WAAW,GAAG,OAAO,CAAC;QACvB,CAAC;IACF,CAAC;SAAM,CAAC;QACP,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,QAAQ,GAAS,WAAW,CAAC;IACjC,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,CAAC;QACnF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAEpC,IAAI,QAAQ,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC1B,SAAS;QACV,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAElE,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QACtD,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;YACvB,SAAS;QACV,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;YACrC,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM;QACP,CAAC;IACF,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3C,IAAI,CAAC,iBAAiB,CAAC,IAAY,CAAC,EAAE,CAAC;YACtC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAS,CAAC;IAEX,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}
+{"version":3,"file":"readability.js","sourceRoot":"","sources":["../../../src/extract-hast/readability.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAU9E,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;AAE5G,MAAM,OAAO,GAAG;IACf,MAAM,EAAE,kCAAkC;IAC1C,MAAM,EAAE,4CAA4C;IACpD,yBAAyB,EAAE,8BAA8B;IACzD,kBAAkB,EACjB,yQAAyQ;IAC1Q,oBAAoB,EAAE,mDAAmD;CACzE,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;AAE/G,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEpF,MAAM,mBAAmB,GAAG;IAC3B,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;CACP,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,IAAU,EAAE,EAAE;IACjC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO;IACR,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAChC,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAChC,OAAO;IACR,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IAC3E,IAAI,QAAQ,EAAE,CAAC;QACd,OAAO,QAAkB,CAAC;IAC3B,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,IAAU,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEnF,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC7C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,8BAA8B;IAC9B,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yCAAyC;IACzC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC7E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QAChG,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,IAAc,CAAC,EAAE,CAAC;QAC3F,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,6BAA6B;IAC7B,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3E,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAEnC,IAAI,OAAO,CAAC,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,6BAA6B;IAC7B,IAAI,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzF,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE;IACxC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAE7B,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACd,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;IACpE,MAAM,YAAY,GAAG,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;IAEpE,OAAO,WAAW,KAAK,YAAY,IAAI,WAAW,CAAC;AACpD,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,IAAU,EAAE,EAAE;IACxC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACtB,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAQ,EAAE;IAC1D,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;IAE1C,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAA2B,CAAC;IAC5D,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,EAAE;QAChD,IAAI,CAAC,cAAc,CAAC,IAAY,CAAC,EAAE,CAAC;YACnC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,sBAAsB,CAAC,IAAY,CAAC,EAAE,CAAC;YAC3C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,WAAW,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACvC,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAC3C,IAAI,aAAa,GAAG,IAAI,IAAI,mBAAmB,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAwC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACtH,IAAI,QAAQ,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QACrC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,QAAQ,GAAS,WAAW,CAAC;IACjC,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,CAAC;QACnF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAEpC,IAAI,QAAQ,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC1B,SAAS;QACV,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAElE,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QACtD,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;YACvB,SAAS;QACV,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;YACrC,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM;QACP,CAAC;IACF,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3C,IAAI,CAAC,iBAAiB,CAAC,IAAY,CAAC,EAAE,CAAC;YACtC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,qBAAqB,CAAC,IAAY,CAAC,EAAE,CAAC;YAC1C,OAAO,KAAK,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC,CAAS,CAAC;IAEX,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}

package/dist/types/extract-hast/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/extract-hast/utils.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE,CAC/C,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;AAElF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,EAAE;IAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACjD,OAAO,OAAO,CAAC,UAAU,CAAC,SAAqB,CAAC;IACjD,CAAC;IACD,OAAO,EAAE,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,KAAc,EAAE,KAAa,EAAE,EAAE;IAC7D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,OAAgB,EAAE,QAAkB,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE;IAC/E,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC1C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACvC,OAAO,IAAI,CAAC;QACb,CAAC;QACD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/extract-hast/utils.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE,CAC/C,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AAEhF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,EAAE;IAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACjD,OAAO,OAAO,CAAC,UAAU,CAAC,SAAqB,CAAC;IACjD,CAAC;IACD,OAAO,EAAE,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,KAAc,EAAE,KAAa,EAAE,EAAE;IAC7D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,OAAgB,EAAE,QAAkB,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE;IAC/E,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC1C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACvC,OAAO,IAAI,CAAC;QACb,CAAC;QACD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC"}

package/dist/types/extract-mdast.d.ts

@@ -0,0 +1,2 @@
+import type { Nodes as Mdast } from "mdast";
+export declare const extractMdast: (node: Mdast) => Mdast;

package/dist/types/extract-mdast.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-mdast.js","sourceRoot":"","sources":["../../src/extract-mdast.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,eAAe,GAAG,CAAC,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;AAEvE,MAAM,sBAAsB,GAAG,CAAC,IAAW,EAAE,EAAE;IAC9C,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAK,IAAe,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAW,EAAE,EAAE;IAC3C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;QACvC,IAAI,CAAC,sBAAsB,CAAC,IAAa,CAAC,EAAE,CAAC;YAC5C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAC,CAAC;IACH,OAAO,SAAkB,CAAC;AAC3B,CAAC,CAAC"}

package/dist/types/html-to-mdast.js.map

@@ -1 +1 @@
-{"version":3,"file":"html-to-mdast.js","sourceRoot":"","sources":["../../src/html-to-mdast.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAmB,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAgB5D;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,UAAyB,EAAE,OAA4B,EAAS,EAAE;IAC7F,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;IAErC,MAAM,IAAI,GAAG,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAEpG,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAEpD,MAAM,KAAK,GAAG,OAAO,CAAC,aAAa,EAAE;QACpC,QAAQ,EAAE;YACT,IAAI,EAAE,WAAW;YACjB,GAAG,EAAE,gBAAgB;YACrB,GAAG,EAAE,iBAAiB;YACtB,CAAC,EAAE,cAAc,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;YAClD,GAAG,EAAE,gBAAgB,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;YACxD,KAAK,EAAE,kBAAkB,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;YAC3D,EAAE,EAAE,YAAY;SAChB;KACD,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACd,CAAC,CAAC"}
+{"version":3,"file":"html-to-mdast.js","sourceRoot":"","sources":["../../src/html-to-mdast.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAG7C,OAAO,EAAmB,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAgB5D;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,UAAyB,EAAE,OAA4B,EAAS,EAAE;IAC7F,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;IAErC,MAAM,IAAI,GAAG,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAEpG,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAEpD,MAAM,KAAK,GAAG,OAAO,CAAC,aAAa,EAAE;QACpC,QAAQ,EAAE;YACT,IAAI,EAAE,WAAW;YACjB,GAAG,EAAE,gBAAgB;YACrB,GAAG,EAAE,iBAAiB;YACtB,CAAC,EAAE,cAAc,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;YAClD,GAAG,EAAE,gBAAgB,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;YACxD,KAAK,EAAE,kBAAkB,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;YAC3D,EAAE,EAAE,YAAY;SAChB;KACD,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAE3C,OAAO,cAAc,CAAC;AACvB,CAAC,CAAC"}

package/dist/types/loaders/fetch.d.ts

@@ -0,0 +1,9 @@
+export declare const USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
+/**
+ * Useful function for load the HTML of a URL using the Fetch API.
+ * **Not recommended** for use in production environments.
+ * @param url - The URL to load.
+ * @param userAgent - The user agent to use. Default is a Chrome user agent.
+ * @returns The HTML content of the URL.
+ */
+export declare const loadHtml: (url: string, userAgent?: string) => Promise<string>;

package/dist/types/loaders/fetch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../../src/loaders/fetch.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,UAAU,GACtB,iHAAiH,CAAC;AAEnH;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAE,SAAS,GAAG,UAAU,EAAE,EAAE;IACrE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;IAC5E,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC,CAAC"}

package/dist/types/loaders/playwright.d.ts

@@ -1,2 +1,16 @@
-import { type BrowserContext } from "playwright";
-export declare const loadHtml: (url: string, context?: BrowserContext) => Promise<string>;
+import { type Browser, type BrowserContext } from "playwright";
+/**
+ * Useful function for load the HTML of a URL using Playwright.
+ * **Not recommended** for use in production environments.
+ * @param url - The URL to load.
+ * @param context - The Playwright browser context to use. If not provided, a new browser context will be created and closed after loading the URL.
+ * @returns The HTML content of the URL.
+ * @example
+ * ```ts
+ * import { loadHtml } from "webforai/loaders/playwright";
+ *
+ * const html = await loadHtml("https://example.com");
+ * console.log(html);
+ * ```
+ */
+export declare const loadHtml: (url: string, context?: BrowserContext | Browser) => Promise<string>;

package/dist/types/loaders/playwright.js.map

@@ -1 +1 @@
-{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../../src/loaders/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqC,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEzE,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAE,OAAwB,EAAE,EAAE;IACvE,IAAI,QAAQ,GAAmB,IAAI,CAAC;IACpC,IAAI,QAAwB,CAAC;IAC7B,IAAI,OAAO,EAAE,CAAC;QACb,QAAQ,GAAG,OAAO,CAAC;IACpB,CAAC;SAAM,CAAC;QACP,QAAQ,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,QAAQ,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,CAAC;IACxC,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;IACtC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrB,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACzE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAClC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACnB,IAAI,QAAQ,EAAE,CAAC;QACd,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}
+{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../../src/loaders/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqC,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEzE;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAE,OAAkC,EAAE,EAAE;IACjF,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAExE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;IACtC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAClC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IAEnB,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}

package/dist/types/mdast-handlers/custom-a-handler.js.map

@@ -1 +1 @@
-{"version":3,"file":"custom-a-handler.js","sourceRoot":"","sources":["../../../src/mdast-handlers/custom-a-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAE/D,MAAM,CAAC,MAAM,cAAc,GAC1B,CAAC,OAA8B,EAAU,EAAE,CAC3C,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACf,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,IAAI,CAAC,EAAW,CAAC;QAClE,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,eAAe,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC5C,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}
+{"version":3,"file":"custom-a-handler.js","sourceRoot":"","sources":["../../../src/mdast-handlers/custom-a-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAE/D,MAAM,CAAC,MAAM,cAAc,GAC1B,CAAC,OAA8B,EAAU,EAAE,CAC3C,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACf,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACtB,OAAO,SAAS,CAAC;QAClB,CAAC;QAED,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAW,CAAC;QACpD,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,eAAe,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC5C,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}

package/dist/types/mdast-to-markdown.js.map

@@ -1 +1 @@
-{"version":3,"file":"mdast-to-markdown.js","sourceRoot":"","sources":["../../src/mdast-to-markdown.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAqC,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEvF,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAY/C;;GAEG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAA2B;IACxE,UAAU,EAAE,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;CAC/C,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,KAA4B,EAAE,OAAgC,EAAU,EAAE;IACzG,MAAM,EAAE,OAAO,EAAE,GAAG,iBAAiB,EAAE,GAAG,EAAE,GAAG,iCAAiC,EAAE,GAAG,OAAO,EAAE,CAAC;IAE/F,IAAI,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,iBAAiB,CAAC,CAAC;IAE9D,IAAI,OAAO,EAAE,CAAC;QACb,QAAQ,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC"}
+{"version":3,"file":"mdast-to-markdown.js","sourceRoot":"","sources":["../../src/mdast-to-markdown.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAqC,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEvF,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAY/C;;GAEG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAA2B;IACxE,UAAU,EAAE,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;IAC/C,MAAM,EAAE,GAAG;CACX,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,KAA4B,EAAE,OAAgC,EAAU,EAAE;IACzG,MAAM,EAAE,OAAO,EAAE,GAAG,iBAAiB,EAAE,GAAG,EAAE,GAAG,iCAAiC,EAAE,GAAG,OAAO,EAAE,CAAC;IAE/F,IAAI,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,iBAAiB,CAAC,CAAC;IAE9D,IAAI,OAAO,EAAE,CAAC;QACb,QAAQ,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webforai",
-  "version": "1.3.1",
+  "version": "1.3.3",
   "description": "A library that provides a web interface for AI",
   "author": "inaridiy",
   "license": "MIT",
@@ -42,6 +42,11 @@
       "import": "./dist/loaders/playwright.js",
       "require": "./dist/cjs/loaders/playwright.js"
     },
+    "./loaders/fetch": {
+      "types": "./dist/types/loaders/fetch.d.ts",
+      "import": "./dist/loaders/fetch.js",
+      "require": "./dist/cjs/loaders/fetch.js"
+    },
     "./loaders/cf-puppeteer": {
       "types": "./dist/types/loaders/cf-puppeteer.d.ts",
       "import": "./dist/loaders/cf-puppeteer.js",