webforai 2.1.0 → 3.0.0

package/dist/extractors/lib/sanitize.js

@@ -0,0 +1,415 @@
+// src/extractors/lib/sanitize.ts
+import {
+  MetricsCollector as MetricsCollectorCtor,
+  classList,
+  isElement,
+  isTruthyAttribute,
+  matchString,
+  numericProperty,
+  pruneInPlace,
+  stringProperty,
+  walk
+} from "../../utils/hast-fast.js";
+import {
+  CHROME_TAGS,
+  CONTEXTUAL_LANDMARK_TAGS,
+  DROP_WHEN_EMPTY_TAGS,
+  EMBEDDED_CONTENT_TAGS,
+  HIDDEN_CLASS_NAMES,
+  NON_CONTENT_TAGS,
+  REGEXPS,
+  RESPONSIVE_DISPLAY_OVERRIDE,
+  UNLIKELY_ROLES
+} from "./constants.js";
+import { isUiChromeText } from "./ui-chrome.js";
+var INVISIBLE_STYLE = /(^|;)\s*(display\s*:\s*none|visibility\s*:\s*hidden)\s*(;|$)/i;
+var hasHiddenClass = (element) => {
+  const classes = classList(element);
+  if (!classes.some((name) => HIDDEN_CLASS_NAMES.has(name))) {
+    return classes.some((name) => REGEXPS.hidden.test(name));
+  }
+  return !classes.some((name) => RESPONSIVE_DISPLAY_OVERRIDE.test(name));
+};
+var isNonContent = (node) => {
+  if (node.type === "comment" || node.type === "doctype") {
+    return true;
+  }
+  return isElement(node) && NON_CONTENT_TAGS.has(node.tagName);
+};
+var isHidden = (element) => {
+  if (containsMath(element)) {
+    return false;
+  }
+  if (isTruthyAttribute(element, "hidden")) {
+    return true;
+  }
+  const style = stringProperty(element, "style");
+  if (style && INVISIBLE_STYLE.test(style)) {
+    return true;
+  }
+  if (hasHiddenClass(element)) {
+    return true;
+  }
+  const width = numericProperty(element, "data-rwidth");
+  const height = numericProperty(element, "data-rheight");
+  if (width === 0 && height === 0) {
+    return true;
+  }
+  return false;
+};
+var PLACEHOLDER_IMAGE_SRC = /^data:image\/|^about:blank$|spacer\.gif$|blank\.(gif|png)$/i;
+var isPlaceholderImage = (element) => {
+  if (element.tagName === "img") {
+    const src = stringProperty(element, "src");
+    return src !== void 0 && PLACEHOLDER_IMAGE_SRC.test(src);
+  }
+  if (element.tagName !== "picture") {
+    return false;
+  }
+  for (const child of element.children) {
+    if (isElement(child) && child.tagName === "img") {
+      return isPlaceholderImage(child);
+    }
+  }
+  return false;
+};
+var hoistNoscriptImages = (tree) => {
+  const visit = (parent) => {
+    for (let index = 0; index < parent.children.length; index++) {
+      const child = parent.children[index];
+      if (!isElement(child)) {
+        continue;
+      }
+      if (child.tagName !== "noscript") {
+        visit(child);
+        continue;
+      }
+      const hasImage = child.children.some(
+        (node) => isElement(node) && (node.tagName === "img" || node.tagName === "picture")
+      );
+      if (!hasImage) {
+        continue;
+      }
+      const rescued = child.children;
+      parent.children.splice(index, 1, ...rescued);
+      const previous = parent.children[index - 1];
+      if (index > 0 && isElement(previous) && isPlaceholderImage(previous)) {
+        parent.children.splice(index - 1, 1);
+        index -= 1;
+      }
+      index += rescued.length - 1;
+    }
+  };
+  if ("children" in tree) {
+    visit(tree);
+  }
+};
+var containsMath = (element) => {
+  if (element.tagName === "math") {
+    return true;
+  }
+  for (const child of element.children) {
+    if (isElement(child) && containsMath(child)) {
+      return true;
+    }
+  }
+  return false;
+};
+var stripNonContent = (tree) => {
+  hoistNoscriptImages(tree);
+  return pruneInPlace(tree, (node) => {
+    if (isNonContent(node)) {
+      return false;
+    }
+    if (isElement(node) && isHidden(node)) {
+      return false;
+    }
+    return true;
+  });
+};
+var findLandmarkChrome = (tree) => {
+  const chrome = /* @__PURE__ */ new Set();
+  const visit = (node, depth, insideContent) => {
+    if (!("children" in node)) {
+      return;
+    }
+    for (const child of node.children) {
+      if (!isElement(child)) {
+        continue;
+      }
+      const nowInsideContent = insideContent || child.tagName === "article" || child.tagName === "main" || stringProperty(child, "role") === "main";
+      if (CONTEXTUAL_LANDMARK_TAGS.has(child.tagName) && !insideContent && depth <= 3) {
+        chrome.add(child);
+        continue;
+      }
+      visit(child, depth + 1, nowInsideContent);
+    }
+  };
+  visit(tree, 0, false);
+  return chrome;
+};
+var isUnlikelyCandidate = (element) => {
+  if (element.tagName === "article" || element.tagName === "main" || element.tagName === "body") {
+    return false;
+  }
+  const role = stringProperty(element, "role");
+  if (role && UNLIKELY_ROLES.has(role)) {
+    return true;
+  }
+  if (CHROME_TAGS.has(element.tagName)) {
+    return true;
+  }
+  const match = matchString(element);
+  if (REGEXPS.specialUnlikelyCandidates.test(match)) {
+    return true;
+  }
+  for (const name of [...classList(element), stringProperty(element, "id") ?? ""]) {
+    if (REGEXPS.stronglyUnlikely.test(name)) {
+      return true;
+    }
+  }
+  if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
+    return true;
+  }
+  return false;
+};
+var findUnlikelyElements = (tree) => {
+  const landmarks = findLandmarkChrome(tree);
+  const doomed = [];
+  const visit = (node) => {
+    if (!("children" in node)) {
+      return;
+    }
+    for (const child of node.children) {
+      if (!isElement(child)) {
+        continue;
+      }
+      if (landmarks.has(child) || isUnlikelyCandidate(child)) {
+        doomed.push(child);
+        continue;
+      }
+      visit(child);
+    }
+  };
+  visit(tree);
+  return doomed;
+};
+var stripUnlikely = (tree) => {
+  const doomed = new Set(findUnlikelyElements(tree));
+  return pruneInPlace(tree, (node) => !(isElement(node) && doomed.has(node)));
+};
+var isEmbeddedContent = (element) => {
+  if (EMBEDDED_CONTENT_TAGS.has(element.tagName)) {
+    return true;
+  }
+  return element.tagName === "br" || element.tagName === "hr" || element.tagName === "input";
+};
+var collectPreformattedElements = (tree) => {
+  const protectedElements = /* @__PURE__ */ new Set();
+  const markSubtree = (node) => {
+    if (!("children" in node)) {
+      return;
+    }
+    for (const child of node.children) {
+      if (isElement(child)) {
+        protectedElements.add(child);
+        markSubtree(child);
+      }
+    }
+  };
+  walk(tree, (node) => {
+    if (isElement(node) && (node.tagName === "pre" || node.tagName === "code")) {
+      protectedElements.add(node);
+      markSubtree(node);
+      return false;
+    }
+  });
+  return protectedElements;
+};
+var dropEmptyElements = (tree, collector) => {
+  const preformatted = collectPreformattedElements(tree);
+  const visit = (parent) => {
+    for (let index = 0; index < parent.children.length; index++) {
+      const child = parent.children[index];
+      if (!isElement(child)) {
+        continue;
+      }
+      if (preformatted.has(child) || !DROP_WHEN_EMPTY_TAGS.has(child.tagName)) {
+        visit(child);
+        continue;
+      }
+      if (collector.metrics(child).text > 0 || hasEmbeddedDescendant(child)) {
+        visit(child);
+        continue;
+      }
+      if (holdsWhitespace(child)) {
+        parent.children[index] = { type: "text", value: " " };
+        continue;
+      }
+      parent.children.splice(index, 1);
+      index -= 1;
+    }
+  };
+  if ("children" in tree) {
+    visit(tree);
+  }
+  return tree;
+};
+var holdsWhitespace = (element) => {
+  for (const child of element.children) {
+    if (child.type === "text" && /\s/.test(child.value)) {
+      return true;
+    }
+    if (isElement(child) && holdsWhitespace(child)) {
+      return true;
+    }
+  }
+  return false;
+};
+var hasEmbeddedDescendant = (element) => {
+  for (const child of element.children) {
+    if (!isElement(child)) {
+      continue;
+    }
+    if (isEmbeddedContent(child) || hasEmbeddedDescendant(child)) {
+      return true;
+    }
+  }
+  return false;
+};
+var REFERENCE_HEADING = /^(see also|references?|further reading|external links?|bibliography|sources|footnotes|citations|関連項目|参考文献|外部リンク|脚注|出典|参考リンク|参考資料)$/i;
+var MAX_DANGLING_HEADING_LENGTH = 60;
+var isHeadingTag = (element) => /^h[1-6]$/.test(element.tagName);
+var previousElement = (siblings, index) => {
+  for (let cursor = index - 1; cursor >= 0; cursor--) {
+    const sibling = siblings[cursor];
+    if (isElement(sibling)) {
+      return sibling;
+    }
+    if (sibling.type === "text" && sibling.value.trim().length > 0) {
+      return void 0;
+    }
+  }
+  return void 0;
+};
+var condemnLinkBlock = (block, siblings, index, collector, doomed) => {
+  const heading = previousElement(siblings, index);
+  const isHeading = heading !== void 0 && isHeadingTag(heading);
+  if (isHeading && REFERENCE_HEADING.test(elementText(heading).trim())) {
+    return;
+  }
+  if (introducedByColon(siblings, index, collector)) {
+    return;
+  }
+  doomed.push(block);
+  if (isHeading && elementText(heading).trim().length <= MAX_DANGLING_HEADING_LENGTH) {
+    doomed.push(heading);
+  }
+};
+var introducedByColon = (siblings, index, collector) => {
+  const before = previousElement(siblings, index);
+  if (!before || collector.metrics(before).text === 0) {
+    return false;
+  }
+  return /[:：]\s*$/.test(elementText(before).trim());
+};
+var findCleanupTargets = (tree, collector) => {
+  const doomed = [];
+  const visit = (node) => {
+    if (!("children" in node)) {
+      return;
+    }
+    const siblings = node.children;
+    for (let index = 0; index < siblings.length; index++) {
+      const child = siblings[index];
+      if (!isElement(child)) {
+        continue;
+      }
+      if (child.tagName === "article" || child.tagName === "main") {
+        visit(child);
+        continue;
+      }
+      if (isLinkOnlyBlock(child, collector)) {
+        condemnLinkBlock(child, siblings, index, collector, doomed);
+        continue;
+      }
+      if (isMidArticleWidget(child)) {
+        doomed.push(child);
+        continue;
+      }
+      visit(child);
+    }
+  };
+  visit(tree);
+  return doomed;
+};
+var LINK_BLOCK_TAGS = /* @__PURE__ */ new Set(["ul", "ol", "nav", "div", "p"]);
+var MIN_LINK_BLOCK_ANCHORS = 3;
+var isLinkOnlyBlock = (element, collector) => {
+  if (!LINK_BLOCK_TAGS.has(element.tagName)) {
+    return false;
+  }
+  const metrics = collector.metrics(element);
+  if (metrics.text === 0 || metrics.commas > 0) {
+    return false;
+  }
+  if (metrics.link / metrics.text <= 0.9) {
+    return false;
+  }
+  return countAnchors(element) >= MIN_LINK_BLOCK_ANCHORS;
+};
+var countAnchors = (element) => {
+  let count = 0;
+  const stack = [element];
+  while (stack.length > 0 && count < MIN_LINK_BLOCK_ANCHORS) {
+    const node = stack.pop();
+    if (isElement(node) && node.tagName === "a") {
+      count += 1;
+    }
+    if ("children" in node) {
+      for (const child of node.children) {
+        stack.push(child);
+      }
+    }
+  }
+  return count;
+};
+var isMidArticleWidget = (element) => {
+  if (element.tagName === "form" || element.tagName === "fieldset") {
+    return true;
+  }
+  if (element.tagName === "button") {
+    return true;
+  }
+  return isUiChromeText(elementText(element));
+};
+var cleanContent = (tree, collector) => {
+  const doomed = new Set(findCleanupTargets(tree, collector));
+  if (doomed.size > 0) {
+    pruneInPlace(tree, (node) => !(isElement(node) && doomed.has(node)));
+  }
+  return dropEmptyElements(tree, new MetricsCollectorCtor());
+};
+var elementText = (node) => {
+  if (node.type === "text") {
+    return node.value;
+  }
+  if (!("children" in node)) {
+    return "";
+  }
+  let text = "";
+  for (const child of node.children) {
+    text += elementText(child);
+  }
+  return text;
+};
+export {
+  cleanContent,
+  dropEmptyElements,
+  findCleanupTargets,
+  findLandmarkChrome,
+  findUnlikelyElements,
+  hoistNoscriptImages,
+  stripNonContent,
+  stripUnlikely
+};

package/dist/extractors/lib/score.js

@@ -0,0 +1,242 @@
+// src/extractors/lib/score.ts
+import {
+  classList,
+  isElement,
+  matchString,
+  numericProperty,
+  stringProperty,
+  walk
+} from "../../utils/hast-fast.js";
+import { CANDIDATE_TAGS, CONTENT_ROLES, REGEXPS, SCOREABLE_TAGS, minContentLength } from "./constants.js";
+var TAG_BASE_SCORE = {
+  div: 5,
+  pre: 3,
+  td: 3,
+  blockquote: 3,
+  article: 10,
+  main: 10,
+  section: 5,
+  address: -3,
+  ol: -3,
+  ul: -3,
+  dl: -3,
+  dd: -3,
+  dt: -3,
+  li: -3,
+  form: -3,
+  h1: -5,
+  h2: -5,
+  h3: -5,
+  h4: -5,
+  h5: -5,
+  h6: -5,
+  th: -5
+};
+var MAX_ANCESTOR_DEPTH = 5;
+var MIN_SCOREABLE_TEXT = 25;
+var buildScoringContext = (root, collector, lang) => {
+  const parents = /* @__PURE__ */ new WeakMap();
+  const scoreable = [];
+  let hasGeometry = false;
+  walk(root, (node, parent) => {
+    if (parent) {
+      parents.set(node, parent);
+    }
+    if (!isElement(node)) {
+      return;
+    }
+    if (!hasGeometry && numericProperty(node, "data-rheight") !== void 0) {
+      hasGeometry = true;
+    }
+    if (isParagraphLike(node)) {
+      scoreable.push(node);
+    }
+  });
+  return { collector, parents, lang, hasGeometry, scoreable };
+};
+var classWeight = (element) => {
+  let weight = 0;
+  const id = stringProperty(element, "id") ?? "";
+  const classes = classList(element).join(" ");
+  if (REGEXPS.negative.test(classes)) {
+    weight -= 25;
+  }
+  if (REGEXPS.positive.test(classes)) {
+    weight += 25;
+  }
+  if (REGEXPS.negative.test(id)) {
+    weight -= 25;
+  }
+  if (REGEXPS.positive.test(id)) {
+    weight += 25;
+  }
+  return weight;
+};
+var semanticBonus = (element) => {
+  let bonus = 0;
+  if (element.tagName === "article") {
+    bonus += 60;
+  }
+  if (element.tagName === "main") {
+    bonus += 50;
+  }
+  const role = stringProperty(element, "role");
+  if (role && CONTENT_ROLES.has(role)) {
+    bonus += 40;
+  }
+  const itemprop = stringProperty(element, "itemprop");
+  if (itemprop?.includes("articleBody")) {
+    bonus += 80;
+  }
+  const match = matchString(element);
+  if (/(^|[\s_-])(markdown|prose|article-?content|post-?content|entry-?content|doc-?content|md-?content)([\s_-]|$)/i.test(
+    match
+  )) {
+    bonus += 35;
+  }
+  return bonus;
+};
+var geometryMultiplier = (element, context) => {
+  if (!context.hasGeometry) {
+    return 1;
+  }
+  const height = numericProperty(element, "data-rheight");
+  const width = numericProperty(element, "data-rwidth");
+  if (height === void 0 || width === void 0) {
+    return 1;
+  }
+  if (height === 0 || width === 0) {
+    return 0;
+  }
+  if (width < 220) {
+    return 0.35;
+  }
+  return 1;
+};
+var paragraphScore = (textLength, commas, densityDivisor) => 1 + commas + Math.min(Math.floor(textLength / densityDivisor), 3);
+var ancestorWeight = (level) => {
+  if (level === 0) {
+    return 1;
+  }
+  if (level === 1) {
+    return 2;
+  }
+  return level * 3;
+};
+var isParagraphLike = (element) => {
+  if (!SCOREABLE_TAGS.has(element.tagName)) {
+    return false;
+  }
+  if (element.tagName !== "div" && element.tagName !== "section") {
+    return true;
+  }
+  return !element.children.some((child) => isElement(child) && CANDIDATE_TAGS.has(child.tagName));
+};
+var scoreCandidates = (_root, context) => {
+  const { collector, parents } = context;
+  const scores = /* @__PURE__ */ new Map();
+  const densityDivisor = minContentLength(context.lang) / 4;
+  for (const node of context.scoreable) {
+    const metrics = collector.metrics(node);
+    if (metrics.text < MIN_SCOREABLE_TEXT) {
+      continue;
+    }
+    const contribution = paragraphScore(metrics.text, metrics.commas, densityDivisor);
+    let current = node;
+    for (let level = 0; level < MAX_ANCESTOR_DEPTH && current; level++) {
+      const parent = parents.get(current);
+      if (!(parent && isElement(parent))) {
+        break;
+      }
+      if (CANDIDATE_TAGS.has(parent.tagName) || parent.tagName === "article" || parent.tagName === "main") {
+        const base = scores.get(parent) ?? (TAG_BASE_SCORE[parent.tagName] ?? 0) + classWeight(parent) + semanticBonus(parent);
+        scores.set(parent, base + contribution / ancestorWeight(level));
+      }
+      current = parent;
+    }
+  }
+  const candidates = [];
+  for (const [element, rawScore] of scores) {
+    const density = collector.linkDensity(element);
+    const score = rawScore * (1 - density) * geometryMultiplier(element, context);
+    if (score > 0) {
+      candidates.push({ element, score });
+    }
+  }
+  candidates.sort((a, b) => b.score - a.score);
+  return candidates;
+};
+var collectArticleNodes = (top, candidates, context) => {
+  const { collector, parents } = context;
+  const parent = parents.get(top.element);
+  if (!(parent && isElement(parent))) {
+    return [top.element];
+  }
+  const scoreOf = new Map(candidates.map((candidate) => [candidate.element, candidate.score]));
+  const threshold = Math.max(10, top.score * 0.2);
+  const selected = [];
+  for (const child of parent.children) {
+    if (!isElement(child)) {
+      continue;
+    }
+    if (child === top.element) {
+      selected.push(child);
+      continue;
+    }
+    const siblingScore = scoreOf.get(child) ?? 0;
+    if (siblingScore >= threshold) {
+      selected.push(child);
+      continue;
+    }
+    if (child.tagName === "p" || child.tagName === "section") {
+      const metrics = collector.metrics(child);
+      const density = collector.linkDensity(child);
+      if (metrics.text > 80 && density < 0.25) {
+        selected.push(child);
+      }
+    }
+  }
+  return selected.length > 0 ? selected : [top.element];
+};
+var MAX_CLIMB_HOPS = 5;
+var MIN_CLIMB_RELATIVE_GAIN = 0.08;
+var MIN_CLIMB_ABSOLUTE_GAIN = 500;
+var MAX_CLIMB_GAIN_LINK_DENSITY = 1 / 3;
+var CLIMB_SCORE_FLOOR = 1 / 3;
+var climbToArticleBoundary = (top, candidates, context) => {
+  const { collector, parents } = context;
+  const scoreOf = new Map(candidates.map((candidate) => [candidate.element, candidate.score]));
+  const scoreFloor = top.score * CLIMB_SCORE_FLOOR;
+  let current = top.element;
+  for (let hop = 0; hop < MAX_CLIMB_HOPS; hop++) {
+    const parent = parents.get(current);
+    if (!(parent && isElement(parent))) {
+      break;
+    }
+    const parentScore = scoreOf.get(parent);
+    if (parentScore === void 0 || parentScore < scoreFloor) {
+      break;
+    }
+    const currentMetrics = collector.metrics(current);
+    const parentMetrics = collector.metrics(parent);
+    const gainedText = parentMetrics.text - currentMetrics.text;
+    if (gainedText < Math.max(MIN_CLIMB_ABSOLUTE_GAIN, currentMetrics.text * MIN_CLIMB_RELATIVE_GAIN)) {
+      break;
+    }
+    const gainedLink = parentMetrics.link - currentMetrics.link;
+    if (gainedLink / gainedText > MAX_CLIMB_GAIN_LINK_DENSITY) {
+      break;
+    }
+    current = parent;
+  }
+  return current;
+};
+export {
+  buildScoringContext,
+  classWeight,
+  climbToArticleBoundary,
+  collectArticleNodes,
+  geometryMultiplier,
+  scoreCandidates,
+  semanticBonus
+};