webforai 2.1.0 → 3.0.0

package/dist/cjs/utils/hast-fast.js

@@ -0,0 +1,284 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var hast_fast_exports = {};
+__export(hast_fast_exports, {
+  MetricsCollector: () => MetricsCollector,
+  asRoot: () => asRoot,
+  classList: () => classList,
+  cloneHast: () => cloneHast,
+  collectElements: () => collectElements,
+  findElement: () => findElement,
+  hasProperty: () => hasProperty,
+  isElement: () => isElement,
+  isParent: () => isParent,
+  isTruthyAttribute: () => isTruthyAttribute,
+  matchString: () => matchString,
+  numericProperty: () => numericProperty,
+  property: () => property,
+  pruneInPlace: () => pruneInPlace,
+  stringProperty: () => stringProperty,
+  walk: () => walk
+});
+module.exports = __toCommonJS(hast_fast_exports);
+const EMPTY_METRICS = { text: 0, link: 0, commas: 0, paragraphs: 0, elements: 0 };
+const COMMA_REGEX = /[,、，،]/g;
+const PARAGRAPH_LIKE = /* @__PURE__ */ new Set(["p", "pre", "li", "blockquote", "figcaption", "dd", "dt"]);
+const NON_PROSE_TAGS = /* @__PURE__ */ new Set(["script", "style", "noscript", "template", "svg", "math", "iframe", "object"]);
+const isElement = (node) => typeof node === "object" && node !== null && node.type === "element";
+const isParent = (node) => typeof node === "object" && node !== null && Array.isArray(node.children);
+const classList = (element) => {
+  const className = element.properties?.className;
+  if (Array.isArray(className)) {
+    return className;
+  }
+  if (typeof className === "string") {
+    return className.split(/\s+/).filter(Boolean);
+  }
+  return [];
+};
+const matchString = (element) => {
+  const id = element.properties?.id;
+  const classes = classList(element);
+  if (!id && classes.length === 0) {
+    return element.tagName;
+  }
+  return `${element.tagName} ${typeof id === "string" ? id : ""} ${classes.join(" ")}`;
+};
+class MetricsCollector {
+  #cache = /* @__PURE__ */ new WeakMap();
+  metrics(node) {
+    if (node.type === "text") {
+      const value = node.value;
+      if (value.trim().length === 0) {
+        return EMPTY_METRICS;
+      }
+      return {
+        text: value.length,
+        link: 0,
+        commas: (value.match(COMMA_REGEX) ?? []).length,
+        paragraphs: 0,
+        elements: 0
+      };
+    }
+    if (!isParent(node)) {
+      return EMPTY_METRICS;
+    }
+    const cached = this.#cache.get(node);
+    if (cached) {
+      return cached;
+    }
+    if (isElement(node) && NON_PROSE_TAGS.has(node.tagName)) {
+      this.#cache.set(node, EMPTY_METRICS);
+      return EMPTY_METRICS;
+    }
+    let text = 0;
+    let link = 0;
+    let commas = 0;
+    let paragraphs = 0;
+    let elements = 0;
+    for (const child of node.children) {
+      const childMetrics = this.metrics(child);
+      text += childMetrics.text;
+      link += childMetrics.link;
+      commas += childMetrics.commas;
+      paragraphs += childMetrics.paragraphs;
+      elements += childMetrics.elements;
+      if (isElement(child)) {
+        elements += 1;
+        if (PARAGRAPH_LIKE.has(child.tagName)) {
+          paragraphs += 1;
+        }
+      }
+    }
+    if (isElement(node) && node.tagName === "a") {
+      link = text;
+    }
+    const metrics = { text, link, commas, paragraphs, elements };
+    this.#cache.set(node, metrics);
+    return metrics;
+  }
+  textLength(node) {
+    return this.metrics(node).text;
+  }
+  /** Fraction of a subtree's text that is link anchor text. `0` for empty subtrees. */
+  linkDensity(node) {
+    const { text, link } = this.metrics(node);
+    if (text === 0) {
+      return 0;
+    }
+    return Math.min(1, link / text);
+  }
+}
+const pruneInPlace = (tree, keep) => {
+  if (!isParent(tree)) {
+    return tree;
+  }
+  const stack = [tree];
+  while (stack.length > 0) {
+    const parent = stack.pop();
+    const children = parent.children;
+    let write = 0;
+    for (let read = 0; read < children.length; read++) {
+      const child = children[read];
+      if (!keep(child)) {
+        continue;
+      }
+      children[write++] = child;
+      if (isParent(child)) {
+        stack.push(child);
+      }
+    }
+    children.length = write;
+  }
+  return tree;
+};
+const walk = (tree, visit) => {
+  const stack = [{ node: tree, parent: void 0 }];
+  while (stack.length > 0) {
+    const { node, parent } = stack.pop();
+    if (visit(node, parent) === false) {
+      continue;
+    }
+    if (isParent(node)) {
+      for (let i = node.children.length - 1; i >= 0; i--) {
+        stack.push({ node: node.children[i], parent: node });
+      }
+    }
+  }
+};
+const collectElements = (tree, predicate) => {
+  const found = [];
+  walk(tree, (node) => {
+    if (isElement(node) && predicate(node)) {
+      found.push(node);
+    }
+  });
+  return found;
+};
+const findElement = (tree, predicate) => {
+  let found;
+  walk(tree, (node) => {
+    if (found) {
+      return false;
+    }
+    if (isElement(node) && predicate(node)) {
+      found = node;
+      return false;
+    }
+  });
+  return found;
+};
+const cloneHast = (tree) => {
+  try {
+    return structuredClone(tree);
+  } catch {
+    return manualClone(tree);
+  }
+};
+const manualClone = (node) => {
+  if (Array.isArray(node)) {
+    return node.map(manualClone);
+  }
+  if (typeof node !== "object" || node === null) {
+    return node;
+  }
+  const copy = {};
+  for (const [key, value] of Object.entries(node)) {
+    copy[key] = manualClone(value);
+  }
+  return copy;
+};
+const asRoot = (nodes) => {
+  if (Array.isArray(nodes)) {
+    return { type: "root", children: nodes };
+  }
+  return nodes;
+};
+const property = (element, key) => {
+  const properties = element.properties;
+  if (!properties) {
+    return void 0;
+  }
+  const direct = properties[key];
+  if (direct !== void 0) {
+    return direct;
+  }
+  if (!(key.includes("-") || key.includes(":"))) {
+    return void 0;
+  }
+  return properties[camelCaseAttribute(key)];
+};
+const camelCaseAttribute = (key) => key.replace(/[-:]([a-z])/g, (_, character) => character.toUpperCase());
+const numericProperty = (element, key) => {
+  const raw = property(element, key);
+  if (typeof raw === "number") {
+    return Number.isFinite(raw) ? raw : void 0;
+  }
+  if (typeof raw === "string" && raw.length > 0) {
+    const parsed = Number.parseFloat(raw);
+    return Number.isFinite(parsed) ? parsed : void 0;
+  }
+  return void 0;
+};
+const stringProperty = (element, key) => {
+  const raw = property(element, key);
+  if (typeof raw === "string") {
+    return raw;
+  }
+  if (Array.isArray(raw)) {
+    return raw.join(" ");
+  }
+  if (raw === true) {
+    return "";
+  }
+  return void 0;
+};
+const hasProperty = (element, key) => property(element, key) !== void 0;
+const isTruthyAttribute = (element, key) => {
+  const raw = property(element, key);
+  if (raw === void 0 || raw === null || raw === false) {
+    return false;
+  }
+  if (raw === true) {
+    return true;
+  }
+  if (typeof raw === "string") {
+    return raw.toLowerCase() !== "false";
+  }
+  return true;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  MetricsCollector,
+  asRoot,
+  classList,
+  cloneHast,
+  collectElements,
+  findElement,
+  hasProperty,
+  isElement,
+  isParent,
+  isTruthyAttribute,
+  matchString,
+  numericProperty,
+  property,
+  pruneInPlace,
+  stringProperty,
+  walk
+});

package/dist/constants.js

@@ -1,6 +1,6 @@
 // src/constants.ts
-import { takumiExtractor } from "./extractors/presets/takumi.js";
-var DEFAULT_EXTRACTORS = [takumiExtractor];
+import { autoExtractor } from "./extractors/presets/auto.js";
+var DEFAULT_EXTRACTORS = [autoExtractor];
 export {
   DEFAULT_EXTRACTORS
 };

package/dist/extractors/index.js

@@ -3,7 +3,12 @@ import {
   pipeExtractors
 } from "./pipeExtractors.js";
 import { takumiExtractor } from "./presets/takumi.js";
+import { minimalFilter } from "./presets/minimal-filter.js";
+import { autoExtractor, createAutoExtractor } from "./presets/auto.js";
 export {
+  autoExtractor,
+  createAutoExtractor,
+  minimalFilter,
   pipeExtractors,
   takumiExtractor
 };

package/dist/extractors/lib/constants.js

@@ -0,0 +1,150 @@
+// src/extractors/lib/constants.ts
+var REGEXPS = {
+  /**
+   * Containers that almost never hold the article body.
+   *
+   * Note the words that are deliberately *absent*. `header` and `footer` match the heading
+   * anchors and article furniture that documentation generators emit (`<a class="header">`,
+   * `class="post-header"`), so matching them here deletes the page's headings. The landmark
+   * versions of those elements are handled structurally instead.
+   */
+  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumb|combx|comment(?!ary)|community|cover-wrap|disqus|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|site-index|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar|subscribe|newsletter|paywall|consent|cookie|toolbar|masthead|promo|share|sharing|widget|tooltip|dropdown|modal|overlay|backdrop|carousel|announcement|notification|toast|skip-link|screen-reader|visually-hidden|sr-only/i,
+  /** Rescues containers whose class merely happens to contain an unlikely substring. */
+  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code|post|entry|story|markdown|prose|doc/i,
+  /** Very high-confidence boilerplate, not rescued by the clause above. */
+  stronglyUnlikely: /^(nav|navbar|breadcrumbs?|cookie-?(banner|consent|notice)|consent-?(banner|manager)|skip-?link|site-?header|site-?footer|global-?(header|footer)|social-?(share|links?)|share-?(bar|buttons?)|related-?(posts?|articles?)|newsletter-?(signup|form)|comment-?(form|list)|toc-?sidebar)$/i,
+  /** Class/id fragments that mark real article content. */
+  positive: /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story|markdown|prose|documentation|doc-?content|readme/i,
+  /** Class/id fragments that mark furniture. */
+  negative: /-ad-|hidden|^hid$| hid$|^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget|toolbar|nav|menu|cookie|consent|subscribe|newsletter|popup|modal|overlay|breadcrumb|pagination|social/i,
+  /** Placeholder imagery that should never reach the output. */
+  hidden: /fallback-image/i,
+  byline: /byline|author|dateline|writtenby|p-author|c-author|post-meta|entry-meta/i,
+  /** Wiki/CMS chrome that generic scoring alone tends to keep. */
+  specialUnlikelyCandidates: /frb-|uls-menu|language-link|mw-editsection|noprint|navbox|metadata|catlinks|printfooter/i
+};
+var NON_CONTENT_TAGS = /* @__PURE__ */ new Set([
+  "script",
+  "style",
+  "link",
+  "meta",
+  "noscript",
+  "template",
+  "base",
+  "param",
+  "source",
+  "track",
+  "title"
+]);
+var CHROME_TAGS = /* @__PURE__ */ new Set(["nav", "aside", "dialog", "menu"]);
+var CONTEXTUAL_LANDMARK_TAGS = /* @__PURE__ */ new Set(["header", "footer"]);
+var HIDDEN_CLASS_NAMES = /* @__PURE__ */ new Set([
+  "hidden",
+  "invisible",
+  "is-hidden",
+  "d-none",
+  "display-none",
+  "hide",
+  "is-invisible"
+]);
+var RESPONSIVE_DISPLAY_OVERRIDE = /^(sm|md|lg|xl|2xl|print):(block|flex|grid|inline|inline-block|inline-flex|table|contents|list-item)$/;
+var UNLIKELY_ROLES = /* @__PURE__ */ new Set([
+  "menu",
+  "menubar",
+  "menuitem",
+  "complementary",
+  "navigation",
+  "alert",
+  "alertdialog",
+  "dialog",
+  "banner",
+  "contentinfo",
+  "search",
+  "searchbox",
+  "toolbar",
+  "tablist",
+  "tooltip"
+  // `presentation` is deliberately absent. It marks an element as decorative *structure* — a
+  // layout table, a wrapper — while the content inside it is usually real. Treating it as
+  // chrome deletes article text.
+]);
+var CONTENT_ROLES = /* @__PURE__ */ new Set(["main", "article", "document"]);
+var SCOREABLE_TAGS = /* @__PURE__ */ new Set(["p", "td", "pre", "blockquote", "dd", "li", "figure", "section", "div"]);
+var CANDIDATE_TAGS = /* @__PURE__ */ new Set([
+  "div",
+  "section",
+  "article",
+  "main",
+  "td",
+  "pre",
+  "blockquote",
+  "form",
+  "ul",
+  "ol",
+  "dl"
+]);
+var EMBEDDED_CONTENT_TAGS = /* @__PURE__ */ new Set([
+  "img",
+  "picture",
+  "video",
+  "audio",
+  "iframe",
+  "embed",
+  "object",
+  "svg",
+  // Formulae hold no text of their own, so an empty-wrapper rule would otherwise delete them.
+  "math",
+  "mjx-container",
+  "canvas"
+]);
+var DROP_WHEN_EMPTY_TAGS = /* @__PURE__ */ new Set([
+  "p",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "li",
+  "td",
+  "th",
+  "blockquote",
+  "figcaption",
+  "dd",
+  "dt",
+  "a",
+  "span",
+  "div",
+  "section",
+  "article"
+]);
+var MIN_CONTENT_LENGTH = {
+  ja: 140,
+  zh: 140,
+  ko: 160,
+  th: 160,
+  default: 400
+};
+var minContentLength = (lang) => {
+  if (!lang) {
+    return MIN_CONTENT_LENGTH.default;
+  }
+  const primary = lang.toLowerCase().split("-")[0];
+  return MIN_CONTENT_LENGTH[primary] ?? MIN_CONTENT_LENGTH.default;
+};
+export {
+  CANDIDATE_TAGS,
+  CHROME_TAGS,
+  CONTENT_ROLES,
+  CONTEXTUAL_LANDMARK_TAGS,
+  DROP_WHEN_EMPTY_TAGS,
+  EMBEDDED_CONTENT_TAGS,
+  HIDDEN_CLASS_NAMES,
+  MIN_CONTENT_LENGTH,
+  NON_CONTENT_TAGS,
+  REGEXPS,
+  RESPONSIVE_DISPLAY_OVERRIDE,
+  SCOREABLE_TAGS,
+  UNLIKELY_ROLES,
+  minContentLength
+};