webforai 1.5.1 → 1.6.1

package/dist/cjs/extract-hast/index.js

@@ -30,17 +30,18 @@ const PRESET_EXTRACT_HAST = {
   readability: import_readability.readabilityExtractHast
 };
 const DEFAULT_EXTRACT_HAST = ["readability"];
-const extractHast = (hast, extractors = DEFAULT_EXTRACT_HAST) => {
+const extractHast = (params, extractors = DEFAULT_EXTRACT_HAST) => {
+  const { hast, lang } = params;
   const _extractors = Array.isArray(extractors) ? extractors : [extractors];
   const extracted = _extractors.reduce((acc, extractor) => {
     if (extractor === false) {
       return acc;
     }
     if (typeof extractor === "string" && extractor in PRESET_EXTRACT_HAST) {
-      return PRESET_EXTRACT_HAST[extractor](acc);
+      return PRESET_EXTRACT_HAST[extractor]({ hast: acc, lang });
     }
     if (typeof extractor === "function") {
-      return extractor(acc);
+      return extractor({ hast: acc, lang });
     }
     throw new Error(`Invalid extractor: ${extractor}`);
   }, hast) || hast;

package/dist/cjs/extract-hast/readability.js

@@ -30,35 +30,24 @@ const REGEXPS = {
   hidden: /hidden|invisible|fallback-image/i,
   byline: /byline|author|dateline|writtenby|p-author/i,
   specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
-  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore/i,
+  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
   okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
 };
-const BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry", "main"];
-const PARAGRAPH_TAGS = ["p", "div", "section", "article", "main", "ul", "ol", "li"];
+const BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
+const PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
 const BASE_MINIMAL_LENGTH = {
   ja: 200,
   en: 500
 };
-const tryGetLang = (node) => {
-  if (node.type !== "element") {
-    return;
-  }
-  const element = node;
-  if (element.tagName !== "html") {
-    return;
-  }
-  const langAttr = element.properties.lang || element.properties["xml:lang"];
-  if (langAttr) {
-    return langAttr;
-  }
+const metadataFilter = (node) => {
+  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
 };
-const metadataFilter = (node) => !["comment", "doctype"].includes(node.type);
 const universalElementFilter = (node) => {
   if (node.type !== "element") {
     return true;
   }
   const element = node;
-  if (["script", "aside"].includes(element.tagName)) {
+  if (["aside", "nav"].includes(element.tagName)) {
     return false;
   }
   if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
@@ -89,10 +78,7 @@ const unlikelyElementFilter = (node) => {
     return true;
   }
   const element = node;
-  if (["body", "article", "main", "section", "a"].includes(element.tagName)) {
-    return true;
-  }
-  if ((0, import_utils.hasAncestors)(element, ["table", "code"], 3)) {
+  if (["body", "article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"].includes(element.tagName)) {
     return true;
   }
   const match = (0, import_utils.matchString)(element);
@@ -104,55 +90,36 @@ const unlikelyElementFilter = (node) => {
   }
   return true;
 };
-const isImageLink = (element) => {
-  const a = (0, import_hast_util_select.select)("a", element);
-  const img = (0, import_hast_util_select.select)("img", a);
-  if (!(a && img)) {
-    return false;
-  }
-  const imgFilename = img.properties.src?.toString().split("/").pop();
-  const hrefFilename = a.properties.href?.toString().split("/").pop();
-  return imgFilename === hrefFilename && imgFilename;
-};
-const removeEmptyFilter = (node) => {
+const removeEmptyFilter = (node, _lang) => {
   if (node.type !== "element") {
     return true;
   }
   const element = node;
-  if (!PARAGRAPH_TAGS.includes(element.tagName)) {
+  if (PARAGRAPH_TAGS.includes(element.tagName)) {
     return true;
   }
-  if (isImageLink(element)) {
-    return true;
-  }
-  const text = (0, import_hast_util_to_string.toString)(element);
-  if (text.length < 10) {
+  if (element.tagName === "img" && !element.properties.src) {
     return false;
   }
   return true;
 };
-const readabilityExtractHast = (hast) => {
-  const lang = String((0, import_hast_util_select.select)("html", hast)?.properties.lang || tryGetLang(hast) || "en");
+const readabilityExtractHast = (params) => {
+  const { hast, lang = "en" } = params;
   const body = (0, import_hast_util_select.select)("body", hast) ?? hast;
-  const bodyText = (0, import_hast_util_to_string.toString)(body);
-  const baseFilterd = (0, import_unist_util_filter.filter)(body, (node) => {
-    if (!metadataFilter(node)) {
-      return false;
-    }
-    if (!universalElementFilter(node)) {
-      return false;
-    }
-    return true;
-  });
-  if (!baseFilterd) {
-    return { type: "root", children: [] };
-  }
-  const baseFilterdText = (0, import_hast_util_to_string.toString)(baseFilterd);
-  let [baseTree, baseText] = baseFilterdText.length > bodyText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [body, bodyText];
+  const metadataFilteredHast = (0, import_unist_util_filter.filter)(body, (node) => metadataFilter(node));
+  const metadataFilteredHastText = metadataFilteredHast && (0, import_hast_util_to_string.toString)(metadataFilteredHast);
+  if (!(metadataFilteredHast && metadataFilteredHastText)) {
+    return body;
+  }
+  const baseFilterd = (0, import_unist_util_filter.filter)(metadataFilteredHast, (node) => universalElementFilter(node));
+  const baseFilterdText = baseFilterd ? (0, import_hast_util_to_string.toString)(baseFilterd) : "";
+  const [baseTree, baseText] = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [metadataFilteredHast, metadataFilteredHastText];
   let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
   if (baseText.length < minimalLength) {
     minimalLength = Math.max(0, baseText.length - 200);
   }
+  let extractedTree = baseTree;
+  let extractedText = baseText;
   for (const selector of BODY_SELECTORS) {
     const content = { type: "root", children: (0, import_hast_util_select.selectAll)(selector, baseFilterd) };
     const contentText = (0, import_hast_util_to_string.toString)(content);
@@ -161,18 +128,18 @@ const readabilityExtractHast = (hast) => {
     }
     const links = (0, import_hast_util_select.selectAll)("a", content);
     const linkText = links.map((link) => (0, import_hast_util_to_string.toString)(link)).join("");
-    const linkDensity = linkText.length / bodyText.length;
+    const linkDensity = linkText.length / contentText.length;
     if (linkDensity > 0.4) {
       continue;
     }
     if (contentText.length > minimalLength) {
-      baseTree = content;
-      baseText = contentText;
+      extractedTree = content;
+      extractedText = contentText;
       break;
     }
   }
-  const finalFilteredTree = (0, import_unist_util_filter.filter)(baseTree, (node) => {
-    if (!removeEmptyFilter(node)) {
+  const finalFilteredTree = (0, import_unist_util_filter.filter)(extractedTree, (node) => {
+    if (!removeEmptyFilter(node, lang)) {
       return false;
     }
     if (!unlikelyElementFilter(node)) {
@@ -180,7 +147,7 @@ const readabilityExtractHast = (hast) => {
     }
     return true;
   });
-  const finalTree = (0, import_hast_util_to_string.toString)(finalFilteredTree).length > baseText.length / 3 ? finalFilteredTree : baseTree;
+  const finalTree = (0, import_hast_util_to_string.toString)(finalFilteredTree).length > extractedText.length / 3 ? finalFilteredTree : extractedTree;
   return finalTree;
 };
 // Annotate the CommonJS export names for ESM import in node:

package/dist/cjs/extract-hast/strongly.js

@@ -24,7 +24,8 @@ module.exports = __toCommonJS(strongly_exports);
 var import_hast_util_select = require("hast-util-select");
 var import_unist_util_filter = require("unist-util-filter");
 const REMOVE_REGEX = /fallback|icon|header|nav|speechify-ignore|assist|-ad-|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|assist|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i;
-const stronglyExtractHast = (hast) => {
+const stronglyExtractHast = (params) => {
+  const { hast } = params;
   const selectors = ["article", "#article", "#content", "main", "div.container"];
   let selectedElement = hast;
   for (const selector of selectors) {

package/dist/cjs/extract-hast/utils.js

@@ -19,7 +19,6 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var utils_exports = {};
 __export(utils_exports, {
   classnames: () => classnames,
-  hasAncestors: () => hasAncestors,
   isStrInclude: () => isStrInclude,
   matchString: () => matchString
 });
@@ -37,23 +36,9 @@ const isStrInclude = (value, match) => {
   }
   return false;
 };
-const hasAncestors = (element, tagNames, depth = 3) => {
-  let parent = element.parent;
-  for (let i = 0; i < depth; i++) {
-    if (!parent || parent.type !== "element") {
-      return false;
-    }
-    if (tagNames.includes(parent.tagName)) {
-      return true;
-    }
-    parent = parent.parent;
-  }
-  return false;
-};
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   classnames,
-  hasAncestors,
   isStrInclude,
   matchString
 });

package/dist/cjs/html-to-mdast.js

@@ -30,12 +30,13 @@ var import_custom_code_handler = require("./mdast-handlers/custom-code-handler")
 var import_custom_div_handler = require("./mdast-handlers/custom-div-handler");
 var import_custom_img_handler = require("./mdast-handlers/custom-img-handler");
 var import_custom_table_handler = require("./mdast-handlers/custom-table-handler");
-var import_empty_handler = require("./mdast-handlers/empty-handler");
 var import_math_handler = require("./mdast-handlers/math-handler");
+var import_hast_utils = require("./utils/hast-utils");
 const htmlToMdast = (htmlOrHast, options) => {
-  const { extractors } = options || {};
-  const hast = typeof htmlOrHast === "string" ? (0, import_hast_util_from_html.fromHtml)(htmlOrHast, { fragment: true }) : htmlOrHast;
-  const extractedHast = (0, import_extract_hast.extractHast)(hast, extractors);
+  const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
+  const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || (0, import_hast_utils.getLangFromStr)(htmlOrHast), (0, import_hast_util_from_html.fromHtml)(htmlOrHast, { fragment: true })] : [defaultLang || (0, import_hast_utils.getLangFromHast)(htmlOrHast), htmlOrHast];
+  const url = defaultUrl || (0, import_hast_utils.getUrlFromHast)(hast);
+  const extractedHast = (0, import_extract_hast.extractHast)({ hast, lang, url }, extractors);
   const mdast = (0, import_hast_util_to_mdast.toMdast)(extractedHast, {
     handlers: {
       math: import_math_handler.mathHandler,
@@ -43,8 +44,7 @@ const htmlToMdast = (htmlOrHast, options) => {
       pre: import_custom_code_handler.customCodeHandler,
       a: (0, import_custom_a_handler.customAHandler)({ asText: options?.linkAsText }),
       img: (0, import_custom_img_handler.customImgHandler)({ hideImage: options?.hideImage }),
-      table: (0, import_custom_table_handler.customTableHandler)({ asText: options?.tableAsText }),
-      br: import_empty_handler.emptyHandler
+      table: (0, import_custom_table_handler.customTableHandler)({ asText: options?.tableAsText })
     }
   });
   const extractedMdast = (0, import_extract_mdast.extractMdast)(mdast);

package/dist/cjs/link-replacer.js

@@ -21,13 +21,18 @@ __export(link_replacer_exports, {
   linkReplacer: () => linkReplacer
 });
 module.exports = __toCommonJS(link_replacer_exports);
-const linkReplacer = (markdown, link) => {
-  const regex = /(!?\[.*?\]\()(.+?)(\))/g;
-  return markdown.replace(regex, (match, pre, thisUrl, post) => {
-    if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) {
+const linkReplacer = (markdown, base) => {
+  const regex = /(!?\[.*?\]\()([^)\s]+)(\))/g;
+  return markdown.replace(regex, (match, pre, url, post) => {
+    if (/^(https?:|#)/.test(url)) {
+      return match;
+    }
+    try {
+      const absoluteUrl = new URL(url, base).href;
+      return `${pre}${absoluteUrl}${post}`;
+    } catch {
       return match;
     }
-    return `${pre}${new URL(thisUrl, link).href}${post}`;
   });
 };
 // Annotate the CommonJS export names for ESM import in node:

package/dist/cjs/loaders/playwright.js

@@ -21,15 +21,44 @@ __export(playwright_exports, {
   loadHtml: () => loadHtml
 });
 module.exports = __toCommonJS(playwright_exports);
-var import_playwright = require("playwright");
-const loadHtml = async (url, context) => {
-  const _context = context ?? await import_playwright.chromium.launch({ headless: true });
-  const page = await _context.newPage();
-  await page.goto(url, { waitUntil: "load" });
+var import_playwright_core = require("playwright-core");
+const SUPER_BYPASS_DEVICE = {
+  userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+  viewport: { width: 1920, height: 1080 },
+  deviceScaleFactor: 1,
+  hasTouch: false,
+  isMobile: false,
+  javaScriptEnabled: true,
+  locale: "en-US",
+  timezoneId: "America/New_York"
+};
+const loadHtml = async (url, options) => {
+  const { browser, waitUntil, timeout, superBypassMode } = options ?? {};
+  const _browser = browser ?? await import_playwright_core.chromium.launch({ headless: true });
+  const context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : import_playwright_core.devices["Desktop Chrome"]);
+  if (superBypassMode) {
+    await context.addInitScript(() => {
+      Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
+      Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
+      Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
+    });
+  }
+  const page = await context.newPage();
+  if (superBypassMode) {
+    await page.route("**/*.js", (route) => {
+      if (route.request().url().includes("captcha-delivery")) {
+        return route.abort();
+      }
+      return route.continue();
+    });
+  }
+  await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
   const html = await page.content();
   await page.close();
-  if (!context) {
-    await _context.close();
+  if (browser) {
+    await context.close();
+  } else {
+    await _browser.close();
   }
   return html;
 };

package/dist/cjs/{cli/constants.js → mdast-handlers/custom-br-handler.js}

@@ -16,19 +16,16 @@ var __copyProps = (to, from, except, desc) => {
   return to;
 };
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var constants_exports = {};
-__export(constants_exports, {
-  DEFAULT_PATH: () => DEFAULT_PATH,
-  LOADERS: () => LOADERS,
-  MODES: () => MODES
+var custom_br_handler_exports = {};
+__export(custom_br_handler_exports, {
+  customBrHandler: () => customBrHandler
 });
-module.exports = __toCommonJS(constants_exports);
-const DEFAULT_PATH = "https://example.com";
-const LOADERS = ["fetch", "playwright"];
-const MODES = ["default", "ai"];
+module.exports = __toCommonJS(custom_br_handler_exports);
+var import_hast_util_to_mdast = require("hast-util-to-mdast");
+const customBrHandler = (state, node) => {
+  return import_hast_util_to_mdast.defaultHandlers.br(state, node);
+};
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  DEFAULT_PATH,
-  LOADERS,
-  MODES
+  customBrHandler
 });

package/dist/cjs/mdast-to-markdown.js

@@ -33,7 +33,7 @@ const DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
 };
 const mdastToMarkdown = (mdast, options) => {
   const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };
-  let markdown = (0, import_mdast_util_to_markdown.toMarkdown)((0, import_mdast_utils.warpRoot)(mdast), toMarkdownOptions);
+  let markdown = (0, import_mdast_util_to_markdown.toMarkdown)((0, import_mdast_utils.warpRoot)(mdast), toMarkdownOptions).replace(/\*\*\*\*/g, "");
   if (baseUrl) {
     markdown = (0, import_link_replacer.linkReplacer)(markdown, baseUrl);
   }

package/dist/cjs/utils/hast-utils.js

@@ -0,0 +1,71 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var hast_utils_exports = {};
+__export(hast_utils_exports, {
+  getLangFromHast: () => getLangFromHast,
+  getLangFromStr: () => getLangFromStr,
+  getUrlFromHast: () => getUrlFromHast
+});
+module.exports = __toCommonJS(hast_utils_exports);
+var import_hast_util_select = require("hast-util-select");
+const getLangFromHast = (node) => {
+  const html = (0, import_hast_util_select.select)("html", node);
+  if (html && typeof html.properties.lang === "string") {
+    return html.properties.lang;
+  }
+  if (node.type !== "element") {
+    return;
+  }
+  const element = node;
+  if (element.tagName !== "html") {
+    return;
+  }
+  const langAttr = element.properties.lang || element.properties["xml:lang"];
+  if (langAttr) {
+    return langAttr;
+  }
+  return void 0;
+};
+const getLangFromStr = (str) => {
+  const match = str.match(/lang=["']([^"']+)["']/);
+  if (match) {
+    return match[1];
+  }
+  return void 0;
+};
+const getUrlFromHast = (node) => {
+  if (node.type !== "element") {
+    return void 0;
+  }
+  const metaTagAttributes = ["og:url", "twitter:url"];
+  const metaTags = (0, import_hast_util_select.selectAll)("meta", node);
+  for (const meta of metaTags) {
+    const property = meta.properties.property || meta.properties.name;
+    if (typeof property === "string" && metaTagAttributes.includes(property)) {
+      return typeof meta.properties.content === "string" ? meta.properties.content : void 0;
+    }
+  }
+  return void 0;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  getLangFromHast,
+  getLangFromStr,
+  getUrlFromHast
+});

package/dist/extract-hast/index.js

@@ -6,17 +6,18 @@ var PRESET_EXTRACT_HAST = {
   readability: readabilityExtractHast
 };
 var DEFAULT_EXTRACT_HAST = ["readability"];
-var extractHast = (hast, extractors = DEFAULT_EXTRACT_HAST) => {
+var extractHast = (params, extractors = DEFAULT_EXTRACT_HAST) => {
+  const { hast, lang } = params;
   const _extractors = Array.isArray(extractors) ? extractors : [extractors];
   const extracted = _extractors.reduce((acc, extractor) => {
     if (extractor === false) {
       return acc;
     }
     if (typeof extractor === "string" && extractor in PRESET_EXTRACT_HAST) {
-      return PRESET_EXTRACT_HAST[extractor](acc);
+      return PRESET_EXTRACT_HAST[extractor]({ hast: acc, lang });
     }
     if (typeof extractor === "function") {
-      return extractor(acc);
+      return extractor({ hast: acc, lang });
     }
     throw new Error(`Invalid extractor: ${extractor}`);
   }, hast) || hast;

package/dist/extract-hast/readability.js

@@ -2,41 +2,30 @@
 import { select, selectAll } from "hast-util-select";
 import { toString as hastToString } from "hast-util-to-string";
 import { filter } from "unist-util-filter";
-import { classnames, hasAncestors, isStrInclude, matchString } from "./utils.js";
+import { classnames, isStrInclude, matchString } from "./utils.js";
 var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
 var REGEXPS = {
   hidden: /hidden|invisible|fallback-image/i,
   byline: /byline|author|dateline|writtenby|p-author/i,
   specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
-  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore/i,
+  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
   okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
 };
-var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry", "main"];
-var PARAGRAPH_TAGS = ["p", "div", "section", "article", "main", "ul", "ol", "li"];
+var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
+var PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
 var BASE_MINIMAL_LENGTH = {
   ja: 200,
   en: 500
 };
-var tryGetLang = (node) => {
-  if (node.type !== "element") {
-    return;
-  }
-  const element = node;
-  if (element.tagName !== "html") {
-    return;
-  }
-  const langAttr = element.properties.lang || element.properties["xml:lang"];
-  if (langAttr) {
-    return langAttr;
-  }
+var metadataFilter = (node) => {
+  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
 };
-var metadataFilter = (node) => !["comment", "doctype"].includes(node.type);
 var universalElementFilter = (node) => {
   if (node.type !== "element") {
     return true;
   }
   const element = node;
-  if (["script", "aside"].includes(element.tagName)) {
+  if (["aside", "nav"].includes(element.tagName)) {
     return false;
   }
   if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
@@ -67,10 +56,7 @@ var unlikelyElementFilter = (node) => {
     return true;
   }
   const element = node;
-  if (["body", "article", "main", "section", "a"].includes(element.tagName)) {
-    return true;
-  }
-  if (hasAncestors(element, ["table", "code"], 3)) {
+  if (["body", "article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"].includes(element.tagName)) {
     return true;
   }
   const match = matchString(element);
@@ -82,55 +68,36 @@ var unlikelyElementFilter = (node) => {
   }
   return true;
 };
-var isImageLink = (element) => {
-  const a = select("a", element);
-  const img = select("img", a);
-  if (!(a && img)) {
-    return false;
-  }
-  const imgFilename = img.properties.src?.toString().split("/").pop();
-  const hrefFilename = a.properties.href?.toString().split("/").pop();
-  return imgFilename === hrefFilename && imgFilename;
-};
-var removeEmptyFilter = (node) => {
+var removeEmptyFilter = (node, _lang) => {
   if (node.type !== "element") {
     return true;
   }
   const element = node;
-  if (!PARAGRAPH_TAGS.includes(element.tagName)) {
+  if (PARAGRAPH_TAGS.includes(element.tagName)) {
     return true;
   }
-  if (isImageLink(element)) {
-    return true;
-  }
-  const text = hastToString(element);
-  if (text.length < 10) {
+  if (element.tagName === "img" && !element.properties.src) {
     return false;
   }
   return true;
 };
-var readabilityExtractHast = (hast) => {
-  const lang = String(select("html", hast)?.properties.lang || tryGetLang(hast) || "en");
+var readabilityExtractHast = (params) => {
+  const { hast, lang = "en" } = params;
   const body = select("body", hast) ?? hast;
-  const bodyText = hastToString(body);
-  const baseFilterd = filter(body, (node) => {
-    if (!metadataFilter(node)) {
-      return false;
-    }
-    if (!universalElementFilter(node)) {
-      return false;
-    }
-    return true;
-  });
-  if (!baseFilterd) {
-    return { type: "root", children: [] };
-  }
-  const baseFilterdText = hastToString(baseFilterd);
-  let [baseTree, baseText] = baseFilterdText.length > bodyText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [body, bodyText];
+  const metadataFilteredHast = filter(body, (node) => metadataFilter(node));
+  const metadataFilteredHastText = metadataFilteredHast && hastToString(metadataFilteredHast);
+  if (!(metadataFilteredHast && metadataFilteredHastText)) {
+    return body;
+  }
+  const baseFilterd = filter(metadataFilteredHast, (node) => universalElementFilter(node));
+  const baseFilterdText = baseFilterd ? hastToString(baseFilterd) : "";
+  const [baseTree, baseText] = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [metadataFilteredHast, metadataFilteredHastText];
   let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
   if (baseText.length < minimalLength) {
     minimalLength = Math.max(0, baseText.length - 200);
   }
+  let extractedTree = baseTree;
+  let extractedText = baseText;
   for (const selector of BODY_SELECTORS) {
     const content = { type: "root", children: selectAll(selector, baseFilterd) };
     const contentText = hastToString(content);
@@ -139,18 +106,18 @@ var readabilityExtractHast = (hast) => {
     }
     const links = selectAll("a", content);
     const linkText = links.map((link) => hastToString(link)).join("");
-    const linkDensity = linkText.length / bodyText.length;
+    const linkDensity = linkText.length / contentText.length;
     if (linkDensity > 0.4) {
       continue;
     }
     if (contentText.length > minimalLength) {
-      baseTree = content;
-      baseText = contentText;
+      extractedTree = content;
+      extractedText = contentText;
       break;
     }
   }
-  const finalFilteredTree = filter(baseTree, (node) => {
-    if (!removeEmptyFilter(node)) {
+  const finalFilteredTree = filter(extractedTree, (node) => {
+    if (!removeEmptyFilter(node, lang)) {
       return false;
     }
     if (!unlikelyElementFilter(node)) {
@@ -158,7 +125,7 @@ var readabilityExtractHast = (hast) => {
     }
     return true;
   });
-  const finalTree = hastToString(finalFilteredTree).length > baseText.length / 3 ? finalFilteredTree : baseTree;
+  const finalTree = hastToString(finalFilteredTree).length > extractedText.length / 3 ? finalFilteredTree : extractedTree;
   return finalTree;
 };
 export {

package/dist/extract-hast/strongly.js

@@ -2,7 +2,8 @@
 import { select } from "hast-util-select";
 import { filter } from "unist-util-filter";
 var REMOVE_REGEX = /fallback|icon|header|nav|speechify-ignore|assist|-ad-|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|assist|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i;
-var stronglyExtractHast = (hast) => {
+var stronglyExtractHast = (params) => {
+  const { hast } = params;
   const selectors = ["article", "#article", "#content", "main", "div.container"];
   let selectedElement = hast;
   for (const selector of selectors) {