webforai 2.1.0 → 3.0.0

package/dist/cjs/extractors/lib/terminators.js

@@ -0,0 +1,131 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var terminators_exports = {};
+__export(terminators_exports, {
+  TERMINATOR_TAIL_FRACTION: () => TERMINATOR_TAIL_FRACTION,
+  truncateTrailingBoilerplate: () => truncateTrailingBoilerplate
+});
+module.exports = __toCommonJS(terminators_exports);
+var import_hast_fast = require("../../utils/hast-fast");
+const TERMINATOR_PATTERNS = [
+  /^was this (page|article|document) helpful/i,
+  /^thanks for the feedback/i,
+  /^(this page was )?last (modified|updated) (on|at|:)?\s/i,
+  /^learn how to contribute/i,
+  /^help (us )?improve\b/i,
+  // The stranded Yes/No pair of a feedback widget whose question was removed separately.
+  /^yes ?no$/i,
+  /^(related (articles?|posts?|stories|content)|you might also like|recommended for you|more from|read next)$/i,
+  /^(comments?|leave a (comment|reply)|join the (conversation|discussion))$/i,
+  /^(この(ページ|記事)は(参考|役に立ち)ました?か)/,
+  /^(関連記事|あわせて読みたい|こちらもおすすめ|おすすめ記事)$/,
+  /^(コメントを(残す|書く)|コメント一覧)$/,
+  /^最終更新日?[:：\s]/
+];
+const TAIL_START_FRACTION = 0.8;
+const MAX_TERMINATOR_BLOCK_LENGTH = 500;
+const BLOCK_TAGS = /* @__PURE__ */ new Set(["p", "div", "section", "footer", "aside", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol"]);
+const isTerminatorBlock = (element, start, tailStart) => {
+  if (!BLOCK_TAGS.has(element.tagName) || start < tailStart) {
+    return false;
+  }
+  const text = textOf(element).replace(/\s+/g, " ").trim();
+  if (text.length === 0 || text.length > MAX_TERMINATOR_BLOCK_LENGTH) {
+    return false;
+  }
+  return TERMINATOR_PATTERNS.some((pattern) => pattern.test(text));
+};
+const textOf = (node) => {
+  if (node.type === "text") {
+    return node.value;
+  }
+  if (!(0, import_hast_fast.isParent)(node)) {
+    return "";
+  }
+  let text = "";
+  for (const child of node.children) {
+    text += textOf(child);
+  }
+  return text;
+};
+const truncateTrailingBoilerplate = (tree) => {
+  const starts = /* @__PURE__ */ new Map();
+  let total = 0;
+  const measure = (node) => {
+    starts.set(node, total);
+    if (node.type === "text") {
+      total += node.value.trim().length;
+      return;
+    }
+    if ((0, import_hast_fast.isParent)(node)) {
+      for (const child of node.children) {
+        measure(child);
+      }
+    }
+  };
+  measure(tree);
+  if (total === 0) {
+    return false;
+  }
+  const tailStart = total * TAIL_START_FRACTION;
+  let cutoff = -1;
+  const find = (node) => {
+    if (cutoff !== -1 || !(0, import_hast_fast.isParent)(node)) {
+      return;
+    }
+    for (const child of node.children) {
+      if (cutoff !== -1) {
+        return;
+      }
+      if (!(0, import_hast_fast.isElement)(child)) {
+        continue;
+      }
+      const start = starts.get(child) ?? 0;
+      if (isTerminatorBlock(child, start, tailStart)) {
+        cutoff = start;
+        return;
+      }
+      find(child);
+    }
+  };
+  find(tree);
+  if (cutoff === -1) {
+    return false;
+  }
+  (0, import_hast_fast.pruneInPlace)(tree, (node) => {
+    if (node === tree) {
+      return true;
+    }
+    const start = starts.get(node);
+    if (start === void 0) {
+      return true;
+    }
+    if (start >= cutoff) {
+      return false;
+    }
+    return true;
+  });
+  return true;
+};
+const TERMINATOR_TAIL_FRACTION = TAIL_START_FRACTION;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  TERMINATOR_TAIL_FRACTION,
+  truncateTrailingBoilerplate
+});

package/dist/cjs/extractors/lib/ui-chrome.js

@@ -0,0 +1,119 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var ui_chrome_exports = {};
+__export(ui_chrome_exports, {
+  isUiChromeText: () => isUiChromeText
+});
+module.exports = __toCommonJS(ui_chrome_exports);
+const UI_PHRASES = /* @__PURE__ */ new Set([
+  // Documentation and article controls
+  "copy",
+  "copy page",
+  "copy link",
+  "copy code",
+  "copied",
+  "edit this page",
+  "edit on github",
+  "on this page",
+  "in this article",
+  "table of contents",
+  "contents",
+  "print",
+  "print this page",
+  "back to top",
+  "scroll to top",
+  "skip to content",
+  "skip to main content",
+  "permalink",
+  "anchor",
+  // Feedback widgets
+  "was this page helpful",
+  "was this helpful",
+  "was this article helpful",
+  "yes no",
+  "feedback",
+  "send feedback",
+  "report an issue",
+  "rate this page",
+  // Sharing and subscription
+  "share",
+  "share this",
+  "share this article",
+  "share on twitter",
+  "share on facebook",
+  "subscribe",
+  "sign up",
+  "sign in",
+  "log in",
+  "login",
+  "newsletter",
+  // Advertising and promotion
+  "advertisement",
+  "advertisements",
+  "sponsored",
+  "sponsored content",
+  "promoted",
+  // Navigation affordances
+  "read more",
+  "show more",
+  "show less",
+  "see more",
+  "load more",
+  "previous",
+  "next",
+  "previous page",
+  "next page",
+  "menu",
+  "close",
+  "search",
+  // Japanese equivalents
+  "\u76EE\u6B21",
+  "\u5E83\u544A",
+  "\u30B7\u30A7\u30A2",
+  "\u5171\u6709",
+  "\u3053\u306E\u8A18\u4E8B\u3092\u30B7\u30A7\u30A2",
+  "\u7D9A\u304D\u3092\u8AAD\u3080",
+  "\u3082\u3063\u3068\u898B\u308B",
+  "\u95A2\u9023\u8A18\u4E8B",
+  "\u30B9\u30DD\u30F3\u30B5\u30FC\u30EA\u30F3\u30AF",
+  "\u30DA\u30FC\u30B8\u306E\u5148\u982D\u3078",
+  "\u30C8\u30C3\u30D7\u3078\u623B\u308B",
+  "\u524D\u3078",
+  "\u6B21\u3078",
+  "\u9589\u3058\u308B",
+  "\u691C\u7D22",
+  "\u30ED\u30B0\u30A4\u30F3",
+  "\u4F1A\u54E1\u767B\u9332"
+]);
+const MAX_PHRASE_LENGTH = 40;
+const normalize = (text) => text.toLowerCase().replace(/[?!.,:;·・…"'“”‘’()[\]{}<>|/\\—–-]/g, " ").replace(/\s+/g, " ").trim();
+const isUiChromeText = (text) => {
+  if (text.length === 0 || text.length > MAX_PHRASE_LENGTH) {
+    return false;
+  }
+  const normalized = normalize(text);
+  if (normalized.length === 0) {
+    return false;
+  }
+  return UI_PHRASES.has(normalized);
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  isUiChromeText
+});

package/dist/cjs/extractors/pipeExtractors.js

@@ -23,18 +23,21 @@ __export(pipeExtractors_exports, {
 module.exports = __toCommonJS(pipeExtractors_exports);
 var import_constants = require("../constants");
 const pipeExtractors = (params, extractors = import_constants.DEFAULT_EXTRACTORS) => {
-  const { hast, lang } = params;
-  const _extractors = Array.isArray(extractors) ? extractors : [extractors];
-  const extracted = _extractors.reduce((acc, extractor) => {
+  const { hast, lang, url, owned } = params;
+  const list = Array.isArray(extractors) ? extractors : [extractors];
+  let current = hast;
+  let currentOwned = owned ?? false;
+  for (const extractor of list) {
     if (extractor === false) {
-      return acc;
+      continue;
     }
-    if (typeof extractor === "function") {
-      return extractor({ hast: acc, lang });
+    if (typeof extractor !== "function") {
+      throw new Error(`Invalid extractor: ${String(extractor)}`);
     }
-    throw new Error(`Invalid extractor: ${extractor}`);
-  }, hast) || hast;
-  return extracted;
+    current = extractor({ hast: current, lang, url, owned: currentOwned });
+    currentOwned = true;
+  }
+  return current;
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/extractors/presets/auto.js

@@ -0,0 +1,44 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var auto_exports = {};
+__export(auto_exports, {
+  autoExtractor: () => autoExtractor,
+  createAutoExtractor: () => createAutoExtractor
+});
+module.exports = __toCommonJS(auto_exports);
+var import_adapters = require("../../adapters");
+var import_takumi = require("./takumi");
+const createAutoExtractor = (options = {}) => {
+  const { adapters } = options;
+  return (params) => {
+    if (adapters !== false) {
+      const extracted = (0, import_adapters.runAdapters)({ hast: params.hast, url: params.url, lang: params.lang }, adapters ?? void 0);
+      if (extracted) {
+        return extracted;
+      }
+    }
+    return (0, import_takumi.takumiExtractor)(params);
+  };
+};
+const autoExtractor = createAutoExtractor();
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  autoExtractor,
+  createAutoExtractor
+});

package/dist/cjs/extractors/presets/minimal-filter.js

@@ -21,65 +21,44 @@ __export(minimal_filter_exports, {
   minimalFilter: () => minimalFilter
 });
 module.exports = __toCommonJS(minimal_filter_exports);
-var import_hast_util_select = require("hast-util-select");
-var import_hast_util_to_string = require("hast-util-to-string");
-var import_unist_util_filter = require("unist-util-filter");
-var import_utils = require("./utils");
-const UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
-const REGEXPS = {
-  hidden: /hidden|invisible|fallback-image/i,
-  byline: /byline|author|dateline|writtenby|p-author/i,
-  specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
-  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
-  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
-};
-const metadataFilter = (node) => {
-  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
-};
-const universalElementFilter = (node) => {
-  if (node.type !== "element") {
+var import_hast_fast = require("../../utils/hast-fast");
+var import_constants = require("../lib/constants");
+var import_sanitize = require("../lib/sanitize");
+const MIN_RETAINED_FRACTION = 1 / 3;
+const ALWAYS_ACCEPT_LENGTH = 5e3;
+const isChrome = (element) => {
+  if (element.tagName === "aside" || element.tagName === "nav" || element.tagName === "dialog") {
     return true;
   }
-  const element = node;
-  if (["aside", "nav"].includes(element.tagName)) {
-    return false;
-  }
-  if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
-    return false;
-  }
-  if ((0, import_utils.classnames)(element).some((classname) => REGEXPS.hidden.test(classname))) {
-    return false;
-  }
-  if (element.tagName === "dialog") {
-    return false;
-  }
-  if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
-    return false;
+  const role = (0, import_hast_fast.stringProperty)(element, "role");
+  if (role && import_constants.UNLIKELY_ROLES.has(role)) {
+    return true;
   }
-  if (element.properties.rel === "author" && (0, import_utils.isStrInclude)(element.properties.itemprop, "author")) {
-    return false;
+  if (role === "dialog" && (0, import_hast_fast.isTruthyAttribute)(element, "aria-modal")) {
+    return true;
   }
-  if (REGEXPS.byline.test((0, import_utils.matchString)(element))) {
-    return false;
+  if ((0, import_hast_fast.stringProperty)(element, "rel") === "author") {
+    return true;
   }
-  if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
-    return false;
+  if (import_constants.REGEXPS.byline.test((0, import_hast_fast.matchString)(element))) {
+    return true;
   }
-  return true;
+  return false;
 };
 const minimalFilter = (params) => {
   const { hast } = params;
-  const body = (0, import_hast_util_select.select)("body", hast) ?? hast;
-  const metadataFilteredHast = (0, import_unist_util_filter.filter)(body, (node) => metadataFilter(node));
-  const metadataFilteredHastText = metadataFilteredHast && (0, import_hast_util_to_string.toString)(metadataFilteredHast);
-  if (!(metadataFilteredHast && metadataFilteredHastText)) {
+  const body = (0, import_hast_fast.cloneHast)((0, import_hast_fast.findElement)(hast, (element) => element.tagName === "body") ?? hast);
+  (0, import_sanitize.stripNonContent)(body);
+  const collector = new import_hast_fast.MetricsCollector();
+  const baseText = collector.textLength(body);
+  if (baseText === 0) {
     return body;
   }
-  const baseFilterd = (0, import_unist_util_filter.filter)(metadataFilteredHast, (node) => universalElementFilter(node));
-  const baseFilterdText = baseFilterd ? (0, import_hast_util_to_string.toString)(baseFilterd) : "";
-  const isOverExtracted = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3;
-  const baseTree = isOverExtracted && baseFilterd ? baseFilterd : metadataFilteredHast;
-  return baseTree;
+  const filtered = (0, import_hast_fast.cloneHast)(body);
+  (0, import_hast_fast.pruneInPlace)(filtered, (node) => !((0, import_hast_fast.isElement)(node) && isChrome(node)));
+  const filteredText = new import_hast_fast.MetricsCollector().textLength(filtered);
+  const isOverExtracted = filteredText < baseText * MIN_RETAINED_FRACTION && filteredText < ALWAYS_ACCEPT_LENGTH;
+  return isOverExtracted ? body : filtered;
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/extractors/presets/takumi.js

@@ -21,136 +21,98 @@ __export(takumi_exports, {
   takumiExtractor: () => takumiExtractor
 });
 module.exports = __toCommonJS(takumi_exports);
-var import_hast_util_select = require("hast-util-select");
-var import_hast_util_to_string = require("hast-util-to-string");
-var import_unist_util_filter = require("unist-util-filter");
-var import_utils = require("./utils");
-const UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
-const REGEXPS = {
-  hidden: /hidden|invisible|fallback-image/i,
-  byline: /byline|author|dateline|writtenby|p-author/i,
-  specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
-  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
-  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
-};
-const BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
-const PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
-const CONTENTABLE_TAGS = ["article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"];
-const TEXTABLE_TAGS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "ul"];
-const BASE_MINIMAL_LENGTH = { ja: 200, en: 500 };
-const metadataFilter = (node) => {
-  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
-};
-const universalElementFilter = (node) => {
-  if (node.type !== "element") {
-    return true;
-  }
-  const element = node;
-  if (["aside", "nav"].includes(element.tagName)) {
-    return false;
-  }
-  if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
-    return false;
-  }
-  if ((0, import_utils.classnames)(element).some((classname) => REGEXPS.hidden.test(classname))) {
-    return false;
-  }
-  if (element.tagName === "dialog") {
-    return false;
-  }
-  if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
-    return false;
-  }
-  if (element.properties.rel === "author" && (0, import_utils.isStrInclude)(element.properties.itemprop, "author")) {
-    return false;
-  }
-  if (REGEXPS.byline.test((0, import_utils.matchString)(element))) {
-    return false;
-  }
-  if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
-    return false;
+var import_hast_fast = require("../../utils/hast-fast");
+var import_constants = require("../lib/constants");
+var import_sanitize = require("../lib/sanitize");
+var import_score = require("../lib/score");
+var import_terminators = require("../lib/terminators");
+const MIN_RETAINED_FRACTION = 0.25;
+const SEMANTIC_SELECTORS = [
+  (element) => (0, import_hast_fast.stringProperty)(element, "itemprop")?.includes("articleBody") ?? false,
+  (element) => element.tagName === "article",
+  (element) => element.tagName === "main",
+  (element) => (0, import_hast_fast.stringProperty)(element, "role") === "main"
+];
+const findBody = (hast) => (0, import_hast_fast.findElement)(hast, (element) => element.tagName === "body") ?? hast;
+const semanticShortcut = (tree, collector, minLength) => {
+  for (const matches of SEMANTIC_SELECTORS) {
+    const found = [];
+    collectMatching(tree, matches, found);
+    if (found.length !== 1) {
+      continue;
+    }
+    const candidate = found[0];
+    const metrics = collector.metrics(candidate);
+    if (metrics.text >= minLength && collector.linkDensity(candidate) < 0.5) {
+      return candidate;
+    }
   }
-  return true;
+  return void 0;
 };
-const unlikelyElementFilter = (node) => {
-  if (node.type !== "element") {
-    return true;
-  }
-  const element = node;
-  if (CONTENTABLE_TAGS.includes(element.tagName)) {
-    return true;
-  }
-  const match = (0, import_utils.matchString)(element);
-  if (REGEXPS.specialUnlikelyCandidates.test(match)) {
-    return false;
-  }
-  if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
-    return false;
+const collectMatching = (tree, matches, into) => {
+  const stack = [tree];
+  while (stack.length > 0 && into.length < 2) {
+    const node = stack.pop();
+    if ((0, import_hast_fast.isElement)(node) && matches(node)) {
+      into.push(node);
+      continue;
+    }
+    if ("children" in node) {
+      for (const child of node.children) {
+        stack.push(child);
+      }
+    }
   }
-  return true;
 };
-const removeEmptyFilter = (node, _lang) => {
-  if (node.type !== "element") {
-    return true;
-  }
-  const element = node;
-  if (PARAGRAPH_TAGS.includes(element.tagName)) {
-    return true;
-  }
-  if (element.tagName === "img" && !element.properties.src) {
-    return false;
-  }
-  if (TEXTABLE_TAGS.includes(element.tagName) && (0, import_hast_util_to_string.toString)(element).length === 0) {
-    return false;
-  }
-  return true;
+const asRootOf = (nodes) => ({ type: "root", children: nodes });
+const scoredSelection = (tree, collector, lang) => {
+  const context = (0, import_score.buildScoringContext)(tree, collector, lang);
+  const candidates = (0, import_score.scoreCandidates)(tree, context);
+  const top = candidates[0];
+  if (!top) {
+    return void 0;
+  }
+  const promoted = (0, import_score.climbToArticleBoundary)(top, candidates, context);
+  const anchor = promoted === top.element ? top : { element: promoted, score: top.score };
+  const selected = (0, import_score.collectArticleNodes)(anchor, candidates, context);
+  return selected.length === 1 ? selected[0] : asRootOf(selected);
 };
 const takumiExtractor = (params) => {
-  const { hast, lang = "en" } = params;
-  const body = (0, import_hast_util_select.select)("body", hast) ?? hast;
-  const metadataFilteredHast = (0, import_unist_util_filter.filter)(body, (node) => metadataFilter(node));
-  const metadataFilteredHastText = metadataFilteredHast && (0, import_hast_util_to_string.toString)(metadataFilteredHast);
-  if (!(metadataFilteredHast && metadataFilteredHastText)) {
+  const { hast, lang, owned } = params;
+  const body = owned ? findBody(hast) : (0, import_hast_fast.cloneHast)(findBody(hast));
+  (0, import_sanitize.stripNonContent)(body);
+  let collector = new import_hast_fast.MetricsCollector();
+  const baseText = collector.textLength(body);
+  if (baseText === 0) {
     return body;
   }
-  const baseFilterd = (0, import_unist_util_filter.filter)(metadataFilteredHast, (node) => universalElementFilter(node));
-  const baseFilterdText = baseFilterd ? (0, import_hast_util_to_string.toString)(baseFilterd) : "";
-  const [baseTree, baseText] = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [metadataFilteredHast, metadataFilteredHastText];
-  let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
-  if (baseText.length < minimalLength) {
-    minimalLength = Math.max(0, baseText.length - 200);
-  }
-  let extractedTree = baseTree;
-  let extractedText = baseText;
-  for (const selector of BODY_SELECTORS) {
-    const content = { type: "root", children: (0, import_hast_util_select.selectAll)(selector, baseFilterd) };
-    const contentText = (0, import_hast_util_to_string.toString)(content);
-    if (contentText.length < 25) {
-      continue;
-    }
-    const links = (0, import_hast_util_select.selectAll)("a", content);
-    const linkText = links.map((link) => (0, import_hast_util_to_string.toString)(link)).join("");
-    const linkDensity = linkText.length / contentText.length;
-    if (linkDensity > 0.4) {
-      continue;
-    }
-    if (contentText.length > minimalLength) {
-      extractedTree = content;
-      extractedText = contentText;
-      break;
-    }
-  }
-  const finalFilteredTree = (0, import_unist_util_filter.filter)(extractedTree, (node) => {
-    if (!removeEmptyFilter(node, lang)) {
-      return false;
-    }
-    if (!unlikelyElementFilter(node)) {
-      return false;
-    }
-    return true;
-  });
-  const finalTree = (0, import_hast_util_to_string.toString)(finalFilteredTree).length > extractedText.length / 3 ? finalFilteredTree : extractedTree;
-  return finalTree;
+  const minLength = Math.min((0, import_constants.minContentLength)(lang), Math.max(0, baseText - 200));
+  const shortcut = semanticShortcut(body, collector, minLength);
+  const searchRoot = shortcut ?? body;
+  const searchText = collector.textLength(searchRoot);
+  const doomed = (0, import_sanitize.findUnlikelyElements)(searchRoot);
+  const doomedText = doomed.reduce((sum, element) => sum + collector.metrics(element).text, 0);
+  if (searchText - doomedText > Math.max(minLength, searchText * MIN_RETAINED_FRACTION)) {
+    const doomedSet = new Set(doomed);
+    (0, import_hast_fast.pruneInPlace)(searchRoot, (node) => !((0, import_hast_fast.isElement)(node) && doomedSet.has(node)));
+    collector = new import_hast_fast.MetricsCollector();
+  }
+  const scoringRoot = searchRoot;
+  const scoringRootText = collector.textLength(scoringRoot);
+  const selection = shortcut ? scoringRoot : scoredSelection(scoringRoot, collector, lang) ?? scoringRoot;
+  const selectionText = collector.textLength(selection);
+  const selectionIsSound = selectionText >= minLength || selectionText > scoringRootText * MIN_RETAINED_FRACTION;
+  const chosen = selectionIsSound ? selection : scoringRoot;
+  (0, import_terminators.truncateTrailingBoilerplate)(chosen);
+  const cleanupCollector = new import_hast_fast.MetricsCollector();
+  const chosenText = cleanupCollector.textLength(chosen);
+  const widgets = (0, import_sanitize.findCleanupTargets)(chosen, cleanupCollector);
+  const widgetText = widgets.reduce((sum, element) => sum + cleanupCollector.metrics(element).text, 0);
+  const remainingText = chosenText - widgetText;
+  if (remainingText < chosenText * MIN_RETAINED_FRACTION && remainingText < minLength) {
+    return chosen;
+  }
+  return (0, import_sanitize.cleanContent)(chosen, cleanupCollector);
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {