webforai 3.0.0 → 4.1.0

package/dist/cjs/detect-client-shell.js

@@ -0,0 +1,78 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var detect_client_shell_exports = {};
+__export(detect_client_shell_exports, {
+  SHELL_EMPTY_THRESHOLD: () => SHELL_EMPTY_THRESHOLD,
+  SHELL_SUSPICION_THRESHOLD: () => SHELL_SUSPICION_THRESHOLD,
+  detectClientShell: () => detectClientShell
+});
+module.exports = __toCommonJS(detect_client_shell_exports);
+const SHELL_SUSPICION_THRESHOLD = 300;
+const SHELL_EMPTY_THRESHOLD = 50;
+const BODY_PATTERN = /<body[^>]*>([\s\S]*?)<\/body>/i;
+const HEAD_PATTERN = /<head[^>]*>[\s\S]*?<\/head>/gi;
+const NOSCRIPT_PATTERN = /<noscript[^>]*>([\s\S]*?)<\/noscript>/gi;
+const INVISIBLE_ELEMENT_PATTERN = /<(script|style|template|svg|noscript)[^>]*>[\s\S]*?<\/\1>/gi;
+const COMMENT_PATTERN = /<!--[\s\S]*?-->/g;
+const TAG_PATTERN = /<[^>]+>/g;
+const MOUNT_POINT_PATTERN = /<[a-z][^>]*\sid=["'](?:root|app|__next|___gatsby|__nuxt|q-app|svelte|react-root)["']|<app-root[\s>]|\sdata-reactroot[\s>=]/i;
+const CHALLENGE_PATTERN = /cf-chl|challenge-platform|cf_chl_opt|just a moment|attention required|checking your browser|_incapsula_|px-captcha|datadome/i;
+const ENTITIES = {
+  "&nbsp;": " ",
+  "&amp;": "&",
+  "&lt;": "<",
+  "&gt;": ">",
+  "&quot;": '"',
+  "&#39;": "'"
+};
+const visibleTextOf = (markup) => markup.replace(COMMENT_PATTERN, " ").replace(INVISIBLE_ELEMENT_PATTERN, " ").replace(TAG_PATTERN, " ").replace(/&(?:nbsp|amp|lt|gt|quot|#39);/g, (entity) => ENTITIES[entity] ?? " ").replace(/\s+/g, " ").trim();
+const noscriptMentionsJavascript = (html) => {
+  for (const match of html.matchAll(NOSCRIPT_PATTERN)) {
+    if (/javascript/i.test(match[1] ?? "")) {
+      return true;
+    }
+  }
+  return false;
+};
+const detectClientShell = (html) => {
+  const body = BODY_PATTERN.exec(html)?.[1] ?? html.replace(HEAD_PATTERN, " ");
+  const visibleTextLength = visibleTextOf(body).length;
+  if (visibleTextLength >= SHELL_SUSPICION_THRESHOLD) {
+    return { isShell: false, visibleTextLength };
+  }
+  if (CHALLENGE_PATTERN.test(html)) {
+    return { isShell: true, reason: "anti-bot-challenge", visibleTextLength };
+  }
+  if (MOUNT_POINT_PATTERN.test(body)) {
+    return { isShell: true, reason: "spa-shell", visibleTextLength };
+  }
+  if (noscriptMentionsJavascript(html)) {
+    return { isShell: true, reason: "noscript-only", visibleTextLength };
+  }
+  if (visibleTextLength < SHELL_EMPTY_THRESHOLD) {
+    return { isShell: true, reason: "empty-body", visibleTextLength };
+  }
+  return { isShell: false, visibleTextLength };
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SHELL_EMPTY_THRESHOLD,
+  SHELL_SUSPICION_THRESHOLD,
+  detectClientShell
+});

package/dist/cjs/extract-mdast.js

@@ -32,6 +32,61 @@ const emptyDeclarationFilter = (node) => {
   }
   return true;
 };
+const trimEdge = (children, edge) => {
+  const index = edge === "start" ? 0 : children.length - 1;
+  const node = children[index];
+  if (node?.type !== "text") {
+    return children;
+  }
+  const value = edge === "start" ? node.value.trimStart() : node.value.trimEnd();
+  const copy = [...children];
+  if (value.length === 0) {
+    copy.splice(index, 1);
+  } else {
+    copy[index] = { ...node, value };
+  }
+  return copy;
+};
+const liftDisplayMath = (paragraph) => {
+  if (!paragraph.children.some((child) => child.type === "math")) {
+    return void 0;
+  }
+  const out = [];
+  let run = [];
+  const flush = () => {
+    const trimmed = trimEdge(trimEdge(run, "start"), "end");
+    if (trimmed.length > 0) {
+      out.push({ ...paragraph, children: trimmed });
+    }
+    run = [];
+  };
+  for (const child of paragraph.children) {
+    if (child.type === "math") {
+      flush();
+      out.push(child);
+    } else {
+      run.push(child);
+    }
+  }
+  flush();
+  return out;
+};
+const liftDisplayMathIn = (node) => {
+  if (!("children" in node)) {
+    return;
+  }
+  const parent = node;
+  for (let index = 0; index < parent.children.length; index++) {
+    const child = parent.children[index];
+    const lifted = child.type === "paragraph" ? liftDisplayMath(child) : void 0;
+    if (lifted) {
+      parent.children.splice(index, 1, ...lifted);
+      index += lifted.length - 1;
+      continue;
+    }
+    liftDisplayMathIn(child);
+  }
+};
 const extractMdast = (node) => {
   const extracted = (0, import_unist_util_filter.filter)(node, (node2) => {
     if (!emptyDeclarationFilter(node2)) {
@@ -39,6 +94,9 @@ const extractMdast = (node) => {
     }
     return true;
   });
+  if (extracted) {
+    liftDisplayMathIn(extracted);
+  }
   return extracted;
 };
 // Annotate the CommonJS export names for ESM import in node:

package/dist/cjs/extract-meta-refresh.js

@@ -0,0 +1,71 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var extract_meta_refresh_exports = {};
+__export(extract_meta_refresh_exports, {
+  MAX_META_REFRESH_DELAY_SECONDS: () => MAX_META_REFRESH_DELAY_SECONDS,
+  extractMetaRefresh: () => extractMetaRefresh
+});
+module.exports = __toCommonJS(extract_meta_refresh_exports);
+const MAX_META_REFRESH_DELAY_SECONDS = 10;
+const META_TAG_PATTERN = /<meta\b[^>]*>/gi;
+const HTTP_EQUIV_REFRESH_PATTERN = /\bhttp-equiv\s*=\s*("\s*refresh\s*"|'\s*refresh\s*'|refresh\b)/i;
+const CONTENT_ATTR_PATTERN = /\bcontent\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/i;
+const CONTENT_VALUE_PATTERN = /^\s*(\d+(?:\.\d+)?)\s*[;,]\s*url\s*=\s*(?:"([^"]*)"|'([^']*)'|(.+?))\s*$/i;
+const resolveTarget = (raw, baseUrl) => {
+  try {
+    return baseUrl === void 0 ? new URL(raw) : new URL(raw, baseUrl);
+  } catch {
+    return void 0;
+  }
+};
+const extractMetaRefresh = (html, baseUrl) => {
+  for (const [tag] of html.matchAll(META_TAG_PATTERN)) {
+    if (!HTTP_EQUIV_REFRESH_PATTERN.test(tag)) {
+      continue;
+    }
+    const contentMatch = CONTENT_ATTR_PATTERN.exec(tag);
+    const content = contentMatch?.[1] ?? contentMatch?.[2] ?? contentMatch?.[3];
+    if (!content) {
+      continue;
+    }
+    const value = CONTENT_VALUE_PATTERN.exec(content);
+    if (!value) {
+      continue;
+    }
+    const delaySeconds = Number.parseFloat(value[1] ?? "");
+    const raw = (value[2] ?? value[3] ?? value[4] ?? "").trim();
+    if (!raw || delaySeconds > MAX_META_REFRESH_DELAY_SECONDS) {
+      continue;
+    }
+    const target = resolveTarget(raw, baseUrl);
+    if (!target || target.protocol !== "http:" && target.protocol !== "https:") {
+      continue;
+    }
+    if (baseUrl !== void 0 && resolveTarget(baseUrl, void 0)?.href === target.href) {
+      continue;
+    }
+    return { url: target.href, delaySeconds };
+  }
+  return void 0;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  MAX_META_REFRESH_DELAY_SECONDS,
+  extractMetaRefresh
+});

package/dist/cjs/extractors/index.js

@@ -20,6 +20,8 @@ var extractors_exports = {};
 __export(extractors_exports, {
   autoExtractor: () => import_auto.autoExtractor,
   createAutoExtractor: () => import_auto.createAutoExtractor,
+  createKiwameExtractor: () => import_kiwame.createKiwameExtractor,
+  kiwameExtractor: () => import_kiwame.kiwameExtractor,
   minimalFilter: () => import_minimal_filter.minimalFilter,
   pipeExtractors: () => import_pipeExtractors.pipeExtractors,
   takumiExtractor: () => import_takumi.takumiExtractor
@@ -27,12 +29,15 @@ __export(extractors_exports, {
 module.exports = __toCommonJS(extractors_exports);
 var import_pipeExtractors = require("./pipeExtractors");
 var import_takumi = require("./presets/takumi");
+var import_kiwame = require("./presets/kiwame");
 var import_minimal_filter = require("./presets/minimal-filter");
 var import_auto = require("./presets/auto");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   autoExtractor,
   createAutoExtractor,
+  createKiwameExtractor,
+  kiwameExtractor,
   minimalFilter,
   pipeExtractors,
   takumiExtractor