webforai 2.1.0 → 3.0.0

package/dist/cjs/adapters/sites/youtube.js

@@ -0,0 +1,191 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var youtube_exports = {};
+__export(youtube_exports, {
+  extractAssignedJson: () => extractAssignedJson,
+  isYoutubeElement: () => isYoutubeElement,
+  youtubeAdapter: () => youtubeAdapter,
+  youtubeCaptionTracks: () => youtubeCaptionTracks
+});
+module.exports = __toCommonJS(youtube_exports);
+var import_hast_fast = require("../../utils/hast-fast");
+const findObjectEnd = (source, start) => {
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+  for (let index = start; index < source.length; index++) {
+    const character = source[index];
+    if (escaped) {
+      escaped = false;
+    } else if (character === "\\") {
+      escaped = true;
+    } else if (character === '"') {
+      inString = !inString;
+    } else if (!inString) {
+      if (character === "{") {
+        depth += 1;
+      } else if (character === "}" && --depth === 0) {
+        return index + 1;
+      }
+    }
+  }
+  return -1;
+};
+const extractAssignedJson = (source, name) => {
+  const markerIndex = source.indexOf(`${name} =`);
+  if (markerIndex === -1) {
+    return void 0;
+  }
+  const start = source.indexOf("{", markerIndex);
+  if (start === -1) {
+    return void 0;
+  }
+  const end = findObjectEnd(source, start);
+  if (end === -1) {
+    return void 0;
+  }
+  try {
+    return JSON.parse(source.slice(start, end));
+  } catch {
+    return void 0;
+  }
+};
+const scriptText = (root) => (0, import_hast_fast.collectElements)(root, (element2) => element2.tagName === "script").map(
+  (script) => script.children.map((child) => child.type === "text" ? child.value : "").join("")
+);
+const findPlayerResponse = (root) => {
+  for (const source of scriptText(root)) {
+    if (!source.includes("ytInitialPlayerResponse")) {
+      continue;
+    }
+    const parsed = extractAssignedJson(source, "ytInitialPlayerResponse");
+    if (parsed && typeof parsed === "object") {
+      return parsed;
+    }
+  }
+  return void 0;
+};
+const youtubeCaptionTracks = (root) => {
+  const player = findPlayerResponse(root);
+  const renderer = player?.captions?.playerCaptionsTracklistRenderer;
+  const tracks = renderer?.captionTracks;
+  if (!Array.isArray(tracks)) {
+    return [];
+  }
+  return tracks.flatMap((track) => {
+    const url = typeof track.baseUrl === "string" ? track.baseUrl : void 0;
+    const languageCode = typeof track.languageCode === "string" ? track.languageCode : void 0;
+    if (!(url && languageCode)) {
+      return [];
+    }
+    const name = track.name?.simpleText;
+    return [
+      {
+        url,
+        languageCode,
+        name: typeof name === "string" ? name : void 0,
+        isAutomatic: track.kind === "asr"
+      }
+    ];
+  });
+};
+const text = (value) => ({ type: "text", value });
+const element = (tagName, children) => ({
+  type: "element",
+  tagName,
+  properties: {},
+  children
+});
+const heading = (depth, value) => element(`h${depth}`, [text(value)]);
+const paragraph = (value) => element("p", [text(value)]);
+const descriptionNodes = (description) => description.split(/\n{2,}/).map((block) => block.trim()).filter((block) => block.length > 0).map(
+  (block) => element(
+    "p",
+    block.split("\n").flatMap((line, index) => index === 0 ? [text(line)] : [element("br", []), text(line)])
+  )
+);
+const formatDuration = (seconds) => {
+  const hours = Math.floor(seconds / 3600);
+  const minutes = Math.floor(seconds % 3600 / 60);
+  const remainder = seconds % 60;
+  const pad = (value) => String(value).padStart(2, "0");
+  return hours > 0 ? `${hours}:${pad(minutes)}:${pad(remainder)}` : `${minutes}:${pad(remainder)}`;
+};
+const metadataList = (details) => {
+  const items = [];
+  if (details.author) {
+    items.push(element("li", [text(`Channel: ${details.author}`)]));
+  }
+  const seconds = Number(details.lengthSeconds);
+  if (Number.isFinite(seconds) && seconds > 0) {
+    items.push(element("li", [text(`Duration: ${formatDuration(seconds)}`)]));
+  }
+  const views = Number(details.viewCount);
+  if (Number.isFinite(views) && views > 0) {
+    items.push(element("li", [text(`Views: ${views.toLocaleString("en-US")}`)]));
+  }
+  return items.length > 0 ? element("ul", items) : void 0;
+};
+const youtubeAdapter = {
+  id: "youtube",
+  matches: ({ url }) => {
+    if (!url) {
+      return false;
+    }
+    try {
+      return /(^|\.)(youtube\.com|youtu\.be|youtube-nocookie\.com)$/.test(new URL(url).hostname);
+    } catch {
+      return false;
+    }
+  },
+  // Scanning every script body for the JSON blob would cost a full traversal on every page, so
+  // the fallback keys off markers the signature already collected.
+  matchesFingerprint: ({ signature }) => Boolean(signature?.ids.has("movie_player") || signature?.classes.has("ytd-app")),
+  extract: ({ hast }) => {
+    const player = findPlayerResponse(hast);
+    const details = player?.videoDetails;
+    if (!details?.title) {
+      return void 0;
+    }
+    const children = [heading(1, details.title)];
+    const metadata = metadataList(details);
+    if (metadata) {
+      children.push(metadata);
+    }
+    const description = details.shortDescription?.trim();
+    if (description) {
+      children.push(heading(2, "Description"));
+      children.push(...descriptionNodes(description));
+    }
+    const tracks = youtubeCaptionTracks(hast);
+    if (tracks.length > 0) {
+      const languages = [...new Set(tracks.map((track) => track.languageCode))].sort();
+      children.push(paragraph(`Captions available: ${languages.join(", ")}`));
+    }
+    return { type: "root", children };
+  }
+};
+const isYoutubeElement = import_hast_fast.isElement;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  extractAssignedJson,
+  isYoutubeElement,
+  youtubeAdapter,
+  youtubeCaptionTracks
+});

package/dist/cjs/adapters/types.js

@@ -0,0 +1,16 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var types_exports = {};
+module.exports = __toCommonJS(types_exports);

package/dist/cjs/constants.js

@@ -21,8 +21,8 @@ __export(constants_exports, {
   DEFAULT_EXTRACTORS: () => DEFAULT_EXTRACTORS
 });
 module.exports = __toCommonJS(constants_exports);
-var import_takumi = require("./extractors/presets/takumi");
-const DEFAULT_EXTRACTORS = [import_takumi.takumiExtractor];
+var import_auto = require("./extractors/presets/auto");
+const DEFAULT_EXTRACTORS = [import_auto.autoExtractor];
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   DEFAULT_EXTRACTORS

package/dist/cjs/extractors/index.js

@@ -18,14 +18,22 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var extractors_exports = {};
 __export(extractors_exports, {
+  autoExtractor: () => import_auto.autoExtractor,
+  createAutoExtractor: () => import_auto.createAutoExtractor,
+  minimalFilter: () => import_minimal_filter.minimalFilter,
   pipeExtractors: () => import_pipeExtractors.pipeExtractors,
   takumiExtractor: () => import_takumi.takumiExtractor
 });
 module.exports = __toCommonJS(extractors_exports);
 var import_pipeExtractors = require("./pipeExtractors");
 var import_takumi = require("./presets/takumi");
+var import_minimal_filter = require("./presets/minimal-filter");
+var import_auto = require("./presets/auto");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  autoExtractor,
+  createAutoExtractor,
+  minimalFilter,
   pipeExtractors,
   takumiExtractor
 });

package/dist/cjs/extractors/lib/constants.js

@@ -0,0 +1,186 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var constants_exports = {};
+__export(constants_exports, {
+  CANDIDATE_TAGS: () => CANDIDATE_TAGS,
+  CHROME_TAGS: () => CHROME_TAGS,
+  CONTENT_ROLES: () => CONTENT_ROLES,
+  CONTEXTUAL_LANDMARK_TAGS: () => CONTEXTUAL_LANDMARK_TAGS,
+  DROP_WHEN_EMPTY_TAGS: () => DROP_WHEN_EMPTY_TAGS,
+  EMBEDDED_CONTENT_TAGS: () => EMBEDDED_CONTENT_TAGS,
+  HIDDEN_CLASS_NAMES: () => HIDDEN_CLASS_NAMES,
+  MIN_CONTENT_LENGTH: () => MIN_CONTENT_LENGTH,
+  NON_CONTENT_TAGS: () => NON_CONTENT_TAGS,
+  REGEXPS: () => REGEXPS,
+  RESPONSIVE_DISPLAY_OVERRIDE: () => RESPONSIVE_DISPLAY_OVERRIDE,
+  SCOREABLE_TAGS: () => SCOREABLE_TAGS,
+  UNLIKELY_ROLES: () => UNLIKELY_ROLES,
+  minContentLength: () => minContentLength
+});
+module.exports = __toCommonJS(constants_exports);
+const REGEXPS = {
+  /**
+   * Containers that almost never hold the article body.
+   *
+   * Note the words that are deliberately *absent*. `header` and `footer` match the heading
+   * anchors and article furniture that documentation generators emit (`<a class="header">`,
+   * `class="post-header"`), so matching them here deletes the page's headings. The landmark
+   * versions of those elements are handled structurally instead.
+   */
+  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumb|combx|comment(?!ary)|community|cover-wrap|disqus|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|site-index|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar|subscribe|newsletter|paywall|consent|cookie|toolbar|masthead|promo|share|sharing|widget|tooltip|dropdown|modal|overlay|backdrop|carousel|announcement|notification|toast|skip-link|screen-reader|visually-hidden|sr-only/i,
+  /** Rescues containers whose class merely happens to contain an unlikely substring. */
+  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code|post|entry|story|markdown|prose|doc/i,
+  /** Very high-confidence boilerplate, not rescued by the clause above. */
+  stronglyUnlikely: /^(nav|navbar|breadcrumbs?|cookie-?(banner|consent|notice)|consent-?(banner|manager)|skip-?link|site-?header|site-?footer|global-?(header|footer)|social-?(share|links?)|share-?(bar|buttons?)|related-?(posts?|articles?)|newsletter-?(signup|form)|comment-?(form|list)|toc-?sidebar)$/i,
+  /** Class/id fragments that mark real article content. */
+  positive: /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story|markdown|prose|documentation|doc-?content|readme/i,
+  /** Class/id fragments that mark furniture. */
+  negative: /-ad-|hidden|^hid$| hid$|^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget|toolbar|nav|menu|cookie|consent|subscribe|newsletter|popup|modal|overlay|breadcrumb|pagination|social/i,
+  /** Placeholder imagery that should never reach the output. */
+  hidden: /fallback-image/i,
+  byline: /byline|author|dateline|writtenby|p-author|c-author|post-meta|entry-meta/i,
+  /** Wiki/CMS chrome that generic scoring alone tends to keep. */
+  specialUnlikelyCandidates: /frb-|uls-menu|language-link|mw-editsection|noprint|navbox|metadata|catlinks|printfooter/i
+};
+const NON_CONTENT_TAGS = /* @__PURE__ */ new Set([
+  "script",
+  "style",
+  "link",
+  "meta",
+  "noscript",
+  "template",
+  "base",
+  "param",
+  "source",
+  "track",
+  "title"
+]);
+const CHROME_TAGS = /* @__PURE__ */ new Set(["nav", "aside", "dialog", "menu"]);
+const CONTEXTUAL_LANDMARK_TAGS = /* @__PURE__ */ new Set(["header", "footer"]);
+const HIDDEN_CLASS_NAMES = /* @__PURE__ */ new Set([
+  "hidden",
+  "invisible",
+  "is-hidden",
+  "d-none",
+  "display-none",
+  "hide",
+  "is-invisible"
+]);
+const RESPONSIVE_DISPLAY_OVERRIDE = /^(sm|md|lg|xl|2xl|print):(block|flex|grid|inline|inline-block|inline-flex|table|contents|list-item)$/;
+const UNLIKELY_ROLES = /* @__PURE__ */ new Set([
+  "menu",
+  "menubar",
+  "menuitem",
+  "complementary",
+  "navigation",
+  "alert",
+  "alertdialog",
+  "dialog",
+  "banner",
+  "contentinfo",
+  "search",
+  "searchbox",
+  "toolbar",
+  "tablist",
+  "tooltip"
+  // `presentation` is deliberately absent. It marks an element as decorative *structure* — a
+  // layout table, a wrapper — while the content inside it is usually real. Treating it as
+  // chrome deletes article text.
+]);
+const CONTENT_ROLES = /* @__PURE__ */ new Set(["main", "article", "document"]);
+const SCOREABLE_TAGS = /* @__PURE__ */ new Set(["p", "td", "pre", "blockquote", "dd", "li", "figure", "section", "div"]);
+const CANDIDATE_TAGS = /* @__PURE__ */ new Set([
+  "div",
+  "section",
+  "article",
+  "main",
+  "td",
+  "pre",
+  "blockquote",
+  "form",
+  "ul",
+  "ol",
+  "dl"
+]);
+const EMBEDDED_CONTENT_TAGS = /* @__PURE__ */ new Set([
+  "img",
+  "picture",
+  "video",
+  "audio",
+  "iframe",
+  "embed",
+  "object",
+  "svg",
+  // Formulae hold no text of their own, so an empty-wrapper rule would otherwise delete them.
+  "math",
+  "mjx-container",
+  "canvas"
+]);
+const DROP_WHEN_EMPTY_TAGS = /* @__PURE__ */ new Set([
+  "p",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "li",
+  "td",
+  "th",
+  "blockquote",
+  "figcaption",
+  "dd",
+  "dt",
+  "a",
+  "span",
+  "div",
+  "section",
+  "article"
+]);
+const MIN_CONTENT_LENGTH = {
+  ja: 140,
+  zh: 140,
+  ko: 160,
+  th: 160,
+  default: 400
+};
+const minContentLength = (lang) => {
+  if (!lang) {
+    return MIN_CONTENT_LENGTH.default;
+  }
+  const primary = lang.toLowerCase().split("-")[0];
+  return MIN_CONTENT_LENGTH[primary] ?? MIN_CONTENT_LENGTH.default;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  CANDIDATE_TAGS,
+  CHROME_TAGS,
+  CONTENT_ROLES,
+  CONTEXTUAL_LANDMARK_TAGS,
+  DROP_WHEN_EMPTY_TAGS,
+  EMBEDDED_CONTENT_TAGS,
+  HIDDEN_CLASS_NAMES,
+  MIN_CONTENT_LENGTH,
+  NON_CONTENT_TAGS,
+  REGEXPS,
+  RESPONSIVE_DISPLAY_OVERRIDE,
+  SCOREABLE_TAGS,
+  UNLIKELY_ROLES,
+  minContentLength
+});