webforai 2.1.1 → 4.0.0

package/dist/extractors/lib/score.js

@@ -0,0 +1,242 @@
+// src/extractors/lib/score.ts
+import {
+  classList,
+  isElement,
+  matchString,
+  numericProperty,
+  stringProperty,
+  walk
+} from "../../utils/hast-fast.js";
+import { CANDIDATE_TAGS, CONTENT_ROLES, REGEXPS, SCOREABLE_TAGS, minContentLength } from "./constants.js";
+var TAG_BASE_SCORE = {
+  div: 5,
+  pre: 3,
+  td: 3,
+  blockquote: 3,
+  article: 10,
+  main: 10,
+  section: 5,
+  address: -3,
+  ol: -3,
+  ul: -3,
+  dl: -3,
+  dd: -3,
+  dt: -3,
+  li: -3,
+  form: -3,
+  h1: -5,
+  h2: -5,
+  h3: -5,
+  h4: -5,
+  h5: -5,
+  h6: -5,
+  th: -5
+};
+var MAX_ANCESTOR_DEPTH = 5;
+var MIN_SCOREABLE_TEXT = 25;
+var buildScoringContext = (root, collector, lang) => {
+  const parents = /* @__PURE__ */ new WeakMap();
+  const scoreable = [];
+  let hasGeometry = false;
+  walk(root, (node, parent) => {
+    if (parent) {
+      parents.set(node, parent);
+    }
+    if (!isElement(node)) {
+      return;
+    }
+    if (!hasGeometry && numericProperty(node, "data-rheight") !== void 0) {
+      hasGeometry = true;
+    }
+    if (isParagraphLike(node)) {
+      scoreable.push(node);
+    }
+  });
+  return { collector, parents, lang, hasGeometry, scoreable };
+};
+var classWeight = (element) => {
+  let weight = 0;
+  const id = stringProperty(element, "id") ?? "";
+  const classes = classList(element).join(" ");
+  if (REGEXPS.negative.test(classes)) {
+    weight -= 25;
+  }
+  if (REGEXPS.positive.test(classes)) {
+    weight += 25;
+  }
+  if (REGEXPS.negative.test(id)) {
+    weight -= 25;
+  }
+  if (REGEXPS.positive.test(id)) {
+    weight += 25;
+  }
+  return weight;
+};
+var semanticBonus = (element) => {
+  let bonus = 0;
+  if (element.tagName === "article") {
+    bonus += 60;
+  }
+  if (element.tagName === "main") {
+    bonus += 50;
+  }
+  const role = stringProperty(element, "role");
+  if (role && CONTENT_ROLES.has(role)) {
+    bonus += 40;
+  }
+  const itemprop = stringProperty(element, "itemprop");
+  if (itemprop?.includes("articleBody")) {
+    bonus += 80;
+  }
+  const match = matchString(element);
+  if (/(^|[\s_-])(markdown|prose|article-?content|post-?content|entry-?content|doc-?content|md-?content)([\s_-]|$)/i.test(
+    match
+  )) {
+    bonus += 35;
+  }
+  return bonus;
+};
+var geometryMultiplier = (element, context) => {
+  if (!context.hasGeometry) {
+    return 1;
+  }
+  const height = numericProperty(element, "data-rheight");
+  const width = numericProperty(element, "data-rwidth");
+  if (height === void 0 || width === void 0) {
+    return 1;
+  }
+  if (height === 0 || width === 0) {
+    return 0;
+  }
+  if (width < 220) {
+    return 0.35;
+  }
+  return 1;
+};
+var paragraphScore = (textLength, commas, densityDivisor) => 1 + commas + Math.min(Math.floor(textLength / densityDivisor), 3);
+var ancestorWeight = (level) => {
+  if (level === 0) {
+    return 1;
+  }
+  if (level === 1) {
+    return 2;
+  }
+  return level * 3;
+};
+var isParagraphLike = (element) => {
+  if (!SCOREABLE_TAGS.has(element.tagName)) {
+    return false;
+  }
+  if (element.tagName !== "div" && element.tagName !== "section") {
+    return true;
+  }
+  return !element.children.some((child) => isElement(child) && CANDIDATE_TAGS.has(child.tagName));
+};
+var scoreCandidates = (_root, context) => {
+  const { collector, parents } = context;
+  const scores = /* @__PURE__ */ new Map();
+  const densityDivisor = minContentLength(context.lang) / 4;
+  for (const node of context.scoreable) {
+    const metrics = collector.metrics(node);
+    if (metrics.text < MIN_SCOREABLE_TEXT) {
+      continue;
+    }
+    const contribution = paragraphScore(metrics.text, metrics.commas, densityDivisor);
+    let current = node;
+    for (let level = 0; level < MAX_ANCESTOR_DEPTH && current; level++) {
+      const parent = parents.get(current);
+      if (!(parent && isElement(parent))) {
+        break;
+      }
+      if (CANDIDATE_TAGS.has(parent.tagName) || parent.tagName === "article" || parent.tagName === "main") {
+        const base = scores.get(parent) ?? (TAG_BASE_SCORE[parent.tagName] ?? 0) + classWeight(parent) + semanticBonus(parent);
+        scores.set(parent, base + contribution / ancestorWeight(level));
+      }
+      current = parent;
+    }
+  }
+  const candidates = [];
+  for (const [element, rawScore] of scores) {
+    const density = collector.linkDensity(element);
+    const score = rawScore * (1 - density) * geometryMultiplier(element, context);
+    if (score > 0) {
+      candidates.push({ element, score });
+    }
+  }
+  candidates.sort((a, b) => b.score - a.score);
+  return candidates;
+};
+var collectArticleNodes = (top, candidates, context) => {
+  const { collector, parents } = context;
+  const parent = parents.get(top.element);
+  if (!(parent && isElement(parent))) {
+    return [top.element];
+  }
+  const scoreOf = new Map(candidates.map((candidate) => [candidate.element, candidate.score]));
+  const threshold = Math.max(10, top.score * 0.2);
+  const selected = [];
+  for (const child of parent.children) {
+    if (!isElement(child)) {
+      continue;
+    }
+    if (child === top.element) {
+      selected.push(child);
+      continue;
+    }
+    const siblingScore = scoreOf.get(child) ?? 0;
+    if (siblingScore >= threshold) {
+      selected.push(child);
+      continue;
+    }
+    if (child.tagName === "p" || child.tagName === "section") {
+      const metrics = collector.metrics(child);
+      const density = collector.linkDensity(child);
+      if (metrics.text > 80 && density < 0.25) {
+        selected.push(child);
+      }
+    }
+  }
+  return selected.length > 0 ? selected : [top.element];
+};
+var MAX_CLIMB_HOPS = 5;
+var MIN_CLIMB_RELATIVE_GAIN = 0.08;
+var MIN_CLIMB_ABSOLUTE_GAIN = 500;
+var MAX_CLIMB_GAIN_LINK_DENSITY = 1 / 3;
+var CLIMB_SCORE_FLOOR = 1 / 3;
+var climbToArticleBoundary = (top, candidates, context) => {
+  const { collector, parents } = context;
+  const scoreOf = new Map(candidates.map((candidate) => [candidate.element, candidate.score]));
+  const scoreFloor = top.score * CLIMB_SCORE_FLOOR;
+  let current = top.element;
+  for (let hop = 0; hop < MAX_CLIMB_HOPS; hop++) {
+    const parent = parents.get(current);
+    if (!(parent && isElement(parent))) {
+      break;
+    }
+    const parentScore = scoreOf.get(parent);
+    if (parentScore === void 0 || parentScore < scoreFloor) {
+      break;
+    }
+    const currentMetrics = collector.metrics(current);
+    const parentMetrics = collector.metrics(parent);
+    const gainedText = parentMetrics.text - currentMetrics.text;
+    if (gainedText < Math.max(MIN_CLIMB_ABSOLUTE_GAIN, currentMetrics.text * MIN_CLIMB_RELATIVE_GAIN)) {
+      break;
+    }
+    const gainedLink = parentMetrics.link - currentMetrics.link;
+    if (gainedLink / gainedText > MAX_CLIMB_GAIN_LINK_DENSITY) {
+      break;
+    }
+    current = parent;
+  }
+  return current;
+};
+export {
+  buildScoringContext,
+  classWeight,
+  climbToArticleBoundary,
+  collectArticleNodes,
+  geometryMultiplier,
+  scoreCandidates,
+  semanticBonus
+};

package/dist/extractors/lib/terminators.js

@@ -0,0 +1,107 @@
+// src/extractors/lib/terminators.ts
+import { isElement, isParent, pruneInPlace } from "../../utils/hast-fast.js";
+var TERMINATOR_PATTERNS = [
+  /^was this (page|article|document) helpful/i,
+  /^thanks for the feedback/i,
+  /^(this page was )?last (modified|updated) (on|at|:)?\s/i,
+  /^learn how to contribute/i,
+  /^help (us )?improve\b/i,
+  // The stranded Yes/No pair of a feedback widget whose question was removed separately.
+  /^yes ?no$/i,
+  /^(related (articles?|posts?|stories|content)|you might also like|recommended for you|more from|read next)$/i,
+  /^(comments?|leave a (comment|reply)|join the (conversation|discussion))$/i,
+  /^(この(ページ|記事)は(参考|役に立ち)ました?か)/,
+  /^(関連記事|あわせて読みたい|こちらもおすすめ|おすすめ記事)$/,
+  /^(コメントを(残す|書く)|コメント一覧)$/,
+  /^最終更新日?[:：\s]/
+];
+var TAIL_START_FRACTION = 0.8;
+var MAX_TERMINATOR_BLOCK_LENGTH = 500;
+var BLOCK_TAGS = /* @__PURE__ */ new Set(["p", "div", "section", "footer", "aside", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol"]);
+var isTerminatorBlock = (element, start, tailStart) => {
+  if (!BLOCK_TAGS.has(element.tagName) || start < tailStart) {
+    return false;
+  }
+  const text = textOf(element).replace(/\s+/g, " ").trim();
+  if (text.length === 0 || text.length > MAX_TERMINATOR_BLOCK_LENGTH) {
+    return false;
+  }
+  return TERMINATOR_PATTERNS.some((pattern) => pattern.test(text));
+};
+var textOf = (node) => {
+  if (node.type === "text") {
+    return node.value;
+  }
+  if (!isParent(node)) {
+    return "";
+  }
+  let text = "";
+  for (const child of node.children) {
+    text += textOf(child);
+  }
+  return text;
+};
+var truncateTrailingBoilerplate = (tree) => {
+  const starts = /* @__PURE__ */ new Map();
+  let total = 0;
+  const measure = (node) => {
+    starts.set(node, total);
+    if (node.type === "text") {
+      total += node.value.trim().length;
+      return;
+    }
+    if (isParent(node)) {
+      for (const child of node.children) {
+        measure(child);
+      }
+    }
+  };
+  measure(tree);
+  if (total === 0) {
+    return false;
+  }
+  const tailStart = total * TAIL_START_FRACTION;
+  let cutoff = -1;
+  const find = (node) => {
+    if (cutoff !== -1 || !isParent(node)) {
+      return;
+    }
+    for (const child of node.children) {
+      if (cutoff !== -1) {
+        return;
+      }
+      if (!isElement(child)) {
+        continue;
+      }
+      const start = starts.get(child) ?? 0;
+      if (isTerminatorBlock(child, start, tailStart)) {
+        cutoff = start;
+        return;
+      }
+      find(child);
+    }
+  };
+  find(tree);
+  if (cutoff === -1) {
+    return false;
+  }
+  pruneInPlace(tree, (node) => {
+    if (node === tree) {
+      return true;
+    }
+    const start = starts.get(node);
+    if (start === void 0) {
+      return true;
+    }
+    if (start >= cutoff) {
+      return false;
+    }
+    return true;
+  });
+  return true;
+};
+var TERMINATOR_TAIL_FRACTION = TAIL_START_FRACTION;
+export {
+  TERMINATOR_TAIL_FRACTION,
+  truncateTrailingBoilerplate
+};

package/dist/extractors/lib/ui-chrome.js

@@ -0,0 +1,96 @@
+// src/extractors/lib/ui-chrome.ts
+var UI_PHRASES = /* @__PURE__ */ new Set([
+  // Documentation and article controls
+  "copy",
+  "copy page",
+  "copy link",
+  "copy code",
+  "copied",
+  "edit this page",
+  "edit on github",
+  "on this page",
+  "in this article",
+  "table of contents",
+  "contents",
+  "print",
+  "print this page",
+  "back to top",
+  "scroll to top",
+  "skip to content",
+  "skip to main content",
+  "permalink",
+  "anchor",
+  // Feedback widgets
+  "was this page helpful",
+  "was this helpful",
+  "was this article helpful",
+  "yes no",
+  "feedback",
+  "send feedback",
+  "report an issue",
+  "rate this page",
+  // Sharing and subscription
+  "share",
+  "share this",
+  "share this article",
+  "share on twitter",
+  "share on facebook",
+  "subscribe",
+  "sign up",
+  "sign in",
+  "log in",
+  "login",
+  "newsletter",
+  // Advertising and promotion
+  "advertisement",
+  "advertisements",
+  "sponsored",
+  "sponsored content",
+  "promoted",
+  // Navigation affordances
+  "read more",
+  "show more",
+  "show less",
+  "see more",
+  "load more",
+  "previous",
+  "next",
+  "previous page",
+  "next page",
+  "menu",
+  "close",
+  "search",
+  // Japanese equivalents
+  "\u76EE\u6B21",
+  "\u5E83\u544A",
+  "\u30B7\u30A7\u30A2",
+  "\u5171\u6709",
+  "\u3053\u306E\u8A18\u4E8B\u3092\u30B7\u30A7\u30A2",
+  "\u7D9A\u304D\u3092\u8AAD\u3080",
+  "\u3082\u3063\u3068\u898B\u308B",
+  "\u95A2\u9023\u8A18\u4E8B",
+  "\u30B9\u30DD\u30F3\u30B5\u30FC\u30EA\u30F3\u30AF",
+  "\u30DA\u30FC\u30B8\u306E\u5148\u982D\u3078",
+  "\u30C8\u30C3\u30D7\u3078\u623B\u308B",
+  "\u524D\u3078",
+  "\u6B21\u3078",
+  "\u9589\u3058\u308B",
+  "\u691C\u7D22",
+  "\u30ED\u30B0\u30A4\u30F3",
+  "\u4F1A\u54E1\u767B\u9332"
+]);
+var MAX_PHRASE_LENGTH = 40;
+var normalize = (text) => text.toLowerCase().replace(/[?!.,:;·・…"'“”‘’()[\]{}<>|/\\—–-]/g, " ").replace(/\s+/g, " ").trim();
+var isUiChromeText = (text) => {
+  if (text.length === 0 || text.length > MAX_PHRASE_LENGTH) {
+    return false;
+  }
+  const normalized = normalize(text);
+  if (normalized.length === 0) {
+    return false;
+  }
+  return UI_PHRASES.has(normalized);
+};
+export {
+  isUiChromeText
+};

package/dist/extractors/pipeExtractors.js

@@ -1,18 +1,21 @@
 // src/extractors/pipeExtractors.ts
 import { DEFAULT_EXTRACTORS } from "../constants.js";
 var pipeExtractors = (params, extractors = DEFAULT_EXTRACTORS) => {
-  const { hast, lang } = params;
-  const _extractors = Array.isArray(extractors) ? extractors : [extractors];
-  const extracted = _extractors.reduce((acc, extractor) => {
+  const { hast, lang, url, owned } = params;
+  const list = Array.isArray(extractors) ? extractors : [extractors];
+  let current = hast;
+  let currentOwned = owned ?? false;
+  for (const extractor of list) {
     if (extractor === false) {
-      return acc;
+      continue;
     }
-    if (typeof extractor === "function") {
-      return extractor({ hast: acc, lang });
+    if (typeof extractor !== "function") {
+      throw new Error(`Invalid extractor: ${String(extractor)}`);
     }
-    throw new Error(`Invalid extractor: ${extractor}`);
-  }, hast) || hast;
-  return extracted;
+    current = extractor({ hast: current, lang, url, owned: currentOwned });
+    currentOwned = true;
+  }
+  return current;
 };
 export {
   pipeExtractors

package/dist/extractors/presets/auto.js

@@ -0,0 +1,20 @@
+// src/extractors/presets/auto.ts
+import { runAdapters } from "../../adapters/index.js";
+import { takumiExtractor } from "./takumi.js";
+var createAutoExtractor = (options = {}) => {
+  const { adapters } = options;
+  return (params) => {
+    if (adapters !== false) {
+      const extracted = runAdapters({ hast: params.hast, url: params.url, lang: params.lang }, adapters ?? void 0);
+      if (extracted) {
+        return extracted;
+      }
+    }
+    return takumiExtractor(params);
+  };
+};
+var autoExtractor = createAutoExtractor();
+export {
+  autoExtractor,
+  createAutoExtractor
+};

package/dist/extractors/presets/minimal-filter.js

@@ -1,63 +1,51 @@
 // src/extractors/presets/minimal-filter.ts
-import { select } from "hast-util-select";
-import { toString as hastToString } from "hast-util-to-string";
-import { filter } from "unist-util-filter";
-import { classnames, isStrInclude, matchString } from "./utils.js";
-var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
-var REGEXPS = {
-  hidden: /hidden|invisible|fallback-image/i,
-  byline: /byline|author|dateline|writtenby|p-author/i,
-  specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
-  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
-  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
-};
-var metadataFilter = (node) => {
-  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
-};
-var universalElementFilter = (node) => {
-  if (node.type !== "element") {
+import {
+  MetricsCollector,
+  cloneHast,
+  findElement,
+  isElement,
+  isTruthyAttribute,
+  matchString,
+  pruneInPlace,
+  stringProperty
+} from "../../utils/hast-fast.js";
+import { REGEXPS, UNLIKELY_ROLES } from "../lib/constants.js";
+import { stripNonContent } from "../lib/sanitize.js";
+var MIN_RETAINED_FRACTION = 1 / 3;
+var ALWAYS_ACCEPT_LENGTH = 5e3;
+var isChrome = (element) => {
+  if (element.tagName === "aside" || element.tagName === "nav" || element.tagName === "dialog") {
     return true;
   }
-  const element = node;
-  if (["aside", "nav"].includes(element.tagName)) {
-    return false;
-  }
-  if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
-    return false;
-  }
-  if (classnames(element).some((classname) => REGEXPS.hidden.test(classname))) {
-    return false;
-  }
-  if (element.tagName === "dialog") {
-    return false;
+  const role = stringProperty(element, "role");
+  if (role && UNLIKELY_ROLES.has(role)) {
+    return true;
   }
-  if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
-    return false;
+  if (role === "dialog" && isTruthyAttribute(element, "aria-modal")) {
+    return true;
   }
-  if (element.properties.rel === "author" && isStrInclude(element.properties.itemprop, "author")) {
-    return false;
+  if (stringProperty(element, "rel") === "author") {
+    return true;
   }
   if (REGEXPS.byline.test(matchString(element))) {
-    return false;
-  }
-  if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
-    return false;
+    return true;
   }
-  return true;
+  return false;
 };
 var minimalFilter = (params) => {
   const { hast } = params;
-  const body = select("body", hast) ?? hast;
-  const metadataFilteredHast = filter(body, (node) => metadataFilter(node));
-  const metadataFilteredHastText = metadataFilteredHast && hastToString(metadataFilteredHast);
-  if (!(metadataFilteredHast && metadataFilteredHastText)) {
+  const body = cloneHast(findElement(hast, (element) => element.tagName === "body") ?? hast);
+  stripNonContent(body);
+  const collector = new MetricsCollector();
+  const baseText = collector.textLength(body);
+  if (baseText === 0) {
     return body;
   }
-  const baseFilterd = filter(metadataFilteredHast, (node) => universalElementFilter(node));
-  const baseFilterdText = baseFilterd ? hastToString(baseFilterd) : "";
-  const isOverExtracted = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3;
-  const baseTree = isOverExtracted && baseFilterd ? baseFilterd : metadataFilteredHast;
-  return baseTree;
+  const filtered = cloneHast(body);
+  pruneInPlace(filtered, (node) => !(isElement(node) && isChrome(node)));
+  const filteredText = new MetricsCollector().textLength(filtered);
+  const isOverExtracted = filteredText < baseText * MIN_RETAINED_FRACTION && filteredText < ALWAYS_ACCEPT_LENGTH;
+  return isOverExtracted ? body : filtered;
 };
 export {
   minimalFilter