webforai 3.0.0 → 4.1.0

package/dist/cjs/mdast-handlers/twoslash.js

@@ -0,0 +1,71 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var twoslash_exports = {};
+__export(twoslash_exports, {
+  cleanTwoslash: () => cleanTwoslash
+});
+module.exports = __toCommonJS(twoslash_exports);
+var import_hast_fast = require("../utils/hast-fast");
+var import_pre_text = require("../utils/pre-text");
+const cleanTwoslash = (pre) => {
+  if (!(0, import_hast_fast.classList)(pre).includes("twoslash")) {
+    return void 0;
+  }
+  let language;
+  const clean = (element) => {
+    element.children = element.children.flatMap((child) => {
+      if (!(0, import_hast_fast.isElement)(child)) {
+        return [child];
+      }
+      const classes = (0, import_hast_fast.classList)(child);
+      if (classes.includes("language-id")) {
+        language ||= (0, import_pre_text.preText)(child).trim() || void 0;
+        return [];
+      }
+      if (classes.includes("playground-link") || classes.includes("error-behind")) {
+        return [];
+      }
+      if (classes.includes("error")) {
+        return [errorLine(child)];
+      }
+      clean(child);
+      return [child];
+    });
+  };
+  clean(pre);
+  return language && /^[\w+#.-]+$/.test(language) ? language : void 0;
+};
+const errorLine = (error) => {
+  let code = "";
+  let message = "";
+  for (const child of error.children) {
+    if ((0, import_hast_fast.isElement)(child) && (0, import_hast_fast.classList)(child).includes("code")) {
+      code = (0, import_pre_text.preText)(child).trim();
+    } else {
+      message += (0, import_hast_fast.isElement)(child) ? (0, import_pre_text.preText)(child) : child.type === "text" ? child.value : "";
+    }
+  }
+  const prefix = /^\d+$/.test(code) ? `error TS${code}: ` : "error: ";
+  const lines = `${prefix}${message.trim()}`.split("\n").map((line) => `// ${line}`);
+  return { type: "element", tagName: "div", properties: {}, children: [{ type: "text", value: lines.join("\n") }] };
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  cleanTwoslash
+});

package/dist/cjs/mdast-to-markdown.js

@@ -33,11 +33,8 @@ const DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
 };
 const mdastToMarkdown = (mdast, options) => {
   const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };
-  let markdown = (0, import_mdast_util_to_markdown.toMarkdown)((0, import_mdast_utils.warpRoot)(mdast), toMarkdownOptions).replace(/\*\*\*\*/g, "");
-  if (baseUrl) {
-    markdown = (0, import_link_replacer.linkReplacer)(markdown, baseUrl);
-  }
-  return markdown;
+  const tree = (0, import_mdast_utils.warpRoot)(mdast);
+  return (0, import_mdast_util_to_markdown.toMarkdown)(baseUrl ? (0, import_link_replacer.linkReplacer)(tree, baseUrl) : tree, toMarkdownOptions);
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/metadata/index.js

@@ -116,9 +116,10 @@ const readName = (value) => {
 };
 const asString = (value) => typeof value === "string" && value.length > 0 ? value : void 0;
 const MAX_TITLE_LENGTH = 160;
-const asTitle = (value) => {
+const MAX_DOCUMENT_TITLE_LENGTH = 300;
+const asTitle = (value, maxLength = MAX_TITLE_LENGTH) => {
   const cleaned = clean(value);
-  return cleaned && cleaned.length <= MAX_TITLE_LENGTH ? cleaned : void 0;
+  return cleaned && cleaned.length <= maxLength ? cleaned : void 0;
 };
 const clean = (value) => {
   if (value === void 0) {
@@ -184,7 +185,7 @@ const extractMetadata = (tree) => {
     }
     return void 0;
   };
-  const title = asTitle(get("og:title", "twitter:title")) ?? asTitle(asString(article?.headline)) ?? asTitle(asString(article?.name)) ?? asTitle(documentTitle(tree)) ?? asTitle(firstHeading(tree));
+  const title = asTitle(get("og:title", "twitter:title")) ?? asTitle(asString(article?.headline)) ?? asTitle(asString(article?.name)) ?? asTitle(documentTitle(tree), MAX_DOCUMENT_TITLE_LENGTH) ?? asTitle(firstHeading(tree));
   return {
     title,
     description: clean(get("og:description", "twitter:description", "description")) ?? clean(asString(article?.description)),

package/dist/cjs/normalize/index.js

@@ -98,14 +98,16 @@ const katexDuplicates = (node) => {
   if (!(classes.includes("katex") || classes.includes("katex-display"))) {
     return [];
   }
-  const hasMathml = node.children.some((child) => (0, import_hast_fast.isElement)(child) && (0, import_hast_fast.classList)(child).includes("katex-mathml"));
+  const hasMathml = node.children.some(
+    (child) => (0, import_hast_fast.isElement)(child) && (0, import_hast_fast.classList)(child).includes("katex-mathml") && hasMathSource(child)
+  );
   if (!hasMathml) {
     return [];
   }
   return node.children.filter((child) => (0, import_hast_fast.isElement)(child) && (0, import_hast_fast.classList)(child).includes("katex-html"));
 };
 const mediawikiDuplicates = (node) => {
-  if (!(0, import_hast_fast.classList)(node).includes("mwe-math-element")) {
+  if (!(0, import_hast_fast.classList)(node).includes("mwe-math-element") || !hasMathSource(node)) {
     return [];
   }
   return node.children.filter(
@@ -116,8 +118,15 @@ const mathjaxDuplicates = (node) => {
   if (node.tagName !== "mjx-container") {
     return [];
   }
+  const hasMathml = node.children.some(
+    (child) => (0, import_hast_fast.isElement)(child) && child.tagName === "mjx-assistive-mml" && hasMathSource(child)
+  );
+  if (!hasMathml) {
+    return [];
+  }
   return node.children.filter((child) => (0, import_hast_fast.isElement)(child) && child.tagName !== "mjx-assistive-mml");
 };
+const hasMathSource = (node) => Boolean((0, import_hast_fast.findElement)(node, (element) => element.tagName === "math" && element.children.length > 0));
 const dedupeRenderedMath = (tree) => {
   const doomed = /* @__PURE__ */ new Set();
   (0, import_hast_fast.walk)(tree, (node) => {

package/dist/cjs/platform/client.js

@@ -50,14 +50,28 @@ const parseRetryAfter = (body, response) => {
   const header = Number(response.headers.get("retry-after"));
   return Number.isFinite(header) && header > 0 ? header : void 0;
 };
-const toApiError = (response, body) => {
+const dashboardHint = (status, dashboardUrl) => {
+  if (status === 401) {
+    return `create or check API keys at ${dashboardUrl}`;
+  }
+  if (status === 402) {
+    return `manage billing, credits and the spend cap at ${dashboardUrl}`;
+  }
+  return void 0;
+};
+const toApiError = (response, body, dashboardUrl) => {
   const envelope = body;
   const code = typeof envelope?.error?.code === "string" ? envelope.error.code : "invalid_response";
-  const message = typeof envelope?.error?.message === "string" ? envelope.error.message : `unexpected response (HTTP ${response.status})`;
+  let message = typeof envelope?.error?.message === "string" ? envelope.error.message : `unexpected response (HTTP ${response.status})`;
+  const hint = code === "invalid_response" ? void 0 : dashboardHint(response.status, dashboardUrl);
+  if (hint && !message.includes("/dashboard")) {
+    message = `${message} (${hint})`;
+  }
   return new import_error.PlatformApiError(code, message, response.status, parseRetryAfter(body, response));
 };
 const createPlatformClient = (options = {}) => {
   const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+  const dashboardUrl = `${baseUrl}/dashboard`;
   const fetchImpl = (0, import_fetch.resolveFetch)(options.fetch);
   const request = async (path, init) => {
     const headers = { accept: "application/json" };
@@ -68,7 +82,7 @@ const createPlatformClient = (options = {}) => {
       if (!options.apiKey) {
         throw new import_error.PlatformApiError(
           "missing_api_key",
-          "This endpoint needs an API key: pass `apiKey` to createPlatformClient (create one on the platform dashboard).",
+          `This endpoint needs an API key: pass \`apiKey\` to createPlatformClient (create one at ${dashboardUrl}; 1,000 free credits/month).`,
           0
         );
       }
@@ -77,18 +91,19 @@ const createPlatformClient = (options = {}) => {
     const response = await fetchImpl(`${baseUrl}${path}`, {
       method: init.method,
       headers,
-      body: init.body === void 0 ? void 0 : JSON.stringify(init.body)
+      body: init.body === void 0 ? void 0 : JSON.stringify(init.body),
+      ...init.signal ? { signal: init.signal } : {}
     });
     const body = await response.json().catch(() => void 0);
     if (!response.ok) {
-      throw toApiError(response, body);
+      throw toApiError(response, body, dashboardUrl);
     }
     if (body === void 0) {
       throw new import_error.PlatformApiError("invalid_response", "response body is not JSON", response.status);
     }
     return body;
   };
-  const getJob = (jobId) => request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: "GET", auth: true });
+  const getJob = (jobId, signal) => request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: "GET", auth: true, signal });
   const getJobResults = (jobId, opts) => {
     const query = opts?.cursor ? `?cursor=${encodeURIComponent(opts.cursor)}` : "";
     return request(`/v1/jobs/${encodeURIComponent(jobId)}/results${query}`, { method: "GET", auth: true });
@@ -104,6 +119,17 @@ const createPlatformClient = (options = {}) => {
     }
     return await response.json();
   };
+  const getJobOrBackOff = async (jobId, signal, interval) => {
+    try {
+      return await getJob(jobId, signal);
+    } catch (error) {
+      if (!(error instanceof import_error.PlatformApiError && error.status === 429)) {
+        throw error;
+      }
+      await sleep(error.retryAfter !== void 0 ? error.retryAfter * 1e3 : interval, signal);
+      return void 0;
+    }
+  };
   return {
     scrape: (opts) => request("/v1/scrape", { method: "POST", body: opts, auth: true }),
     scrapeAsync: (opts) => request("/v1/scrape", { method: "POST", body: { ...opts, async: true }, auth: true }),
@@ -114,17 +140,48 @@ const createPlatformClient = (options = {}) => {
     waitForJob: async (jobId, opts = {}) => {
       const interval = opts.pollIntervalMs ?? 2e3;
       const timeout = opts.timeoutMs ?? 10 * 60 * 1e3;
-      const startedAt = Date.now();
-      for (; ; ) {
-        const status = await getJob(jobId);
-        opts.onStatus?.(status);
-        if (TERMINAL_STATES.has(status.status)) {
-          return status;
-        }
-        if (Date.now() - startedAt + interval > timeout) {
-          throw new import_error.PlatformApiError("poll_timeout", `job ${jobId} still ${status.status} after ${timeout}ms`, 0);
+      if (!Number.isFinite(interval) || interval < 0 || interval > 2147483647) {
+        throw new RangeError("pollIntervalMs must be between 0 and 2147483647");
+      }
+      if (!Number.isFinite(timeout) || timeout <= 0 || timeout > 2147483647) {
+        throw new RangeError("timeoutMs must be greater than 0 and at most 2147483647");
+      }
+      opts.signal?.throwIfAborted();
+      const controller = new AbortController();
+      const { signal } = controller;
+      const onAbort = () => controller.abort(opts.signal?.reason);
+      opts.signal?.addEventListener("abort", onAbort, { once: true });
+      const timer = setTimeout(
+        () => controller.abort(new import_error.PlatformApiError("poll_timeout", `job ${jobId} did not finish within ${timeout}ms`, 0)),
+        timeout
+      );
+      let rejectOnAbort = () => {
+      };
+      const aborted = new Promise((_resolve, reject) => {
+        rejectOnAbort = () => reject(signal.reason);
+        signal.addEventListener("abort", rejectOnAbort, { once: true });
+      });
+      const poll = async () => {
+        for (; ; ) {
+          signal.throwIfAborted();
+          const status = await getJobOrBackOff(jobId, signal, interval);
+          signal.throwIfAborted();
+          if (status === void 0) {
+            continue;
+          }
+          opts.onStatus?.(status);
+          if (TERMINAL_STATES.has(status.status)) {
+            return status;
+          }
+          await sleep(interval, signal);
         }
-        await sleep(interval, opts.signal);
+      };
+      try {
+        return await Promise.race([poll(), aborted]);
+      } finally {
+        clearTimeout(timer);
+        opts.signal?.removeEventListener("abort", onAbort);
+        signal.removeEventListener("abort", rejectOnAbort);
       }
     },
     jobResults: async function* (jobId) {

package/dist/cjs/platform/index.js

@@ -18,10 +18,12 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var platform_exports = {};
 __export(platform_exports, {
+  CRAWL_SITEMAP_MODES: () => import_types.CRAWL_SITEMAP_MODES,
   DEFAULT_BASE_URL: () => import_client.DEFAULT_BASE_URL,
   ENGINES: () => import_types.ENGINES,
   PlatformApiError: () => import_error.PlatformApiError,
   REGIONS: () => import_types.REGIONS,
+  REQUESTED_ENGINES: () => import_types.REQUESTED_ENGINES,
   createPlatformClient: () => import_client.createPlatformClient,
   isStoredPageStub: () => import_types.isStoredPageStub
 });
@@ -31,10 +33,12 @@ var import_error = require("./error");
 var import_types = require("./types");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  CRAWL_SITEMAP_MODES,
   DEFAULT_BASE_URL,
   ENGINES,
   PlatformApiError,
   REGIONS,
+  REQUESTED_ENGINES,
   createPlatformClient,
   isStoredPageStub
 });

package/dist/cjs/platform/types.js

@@ -18,17 +18,23 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var types_exports = {};
 __export(types_exports, {
+  CRAWL_SITEMAP_MODES: () => CRAWL_SITEMAP_MODES,
   ENGINES: () => ENGINES,
   REGIONS: () => REGIONS,
+  REQUESTED_ENGINES: () => REQUESTED_ENGINES,
   isStoredPageStub: () => isStoredPageStub
 });
 module.exports = __toCommonJS(types_exports);
 const ENGINES = ["fetch", "browser", "proxy-fetch", "proxy-browser"];
-const REGIONS = ["auto", "us", "eu", "uk", "jp", "asia"];
+const REQUESTED_ENGINES = [...ENGINES, "auto"];
+const REGIONS = ["auto", "jp"];
+const CRAWL_SITEMAP_MODES = ["skip", "include", "only"];
 const isStoredPageStub = (item) => item.status === "ok" && "resultUrl" in item;
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  CRAWL_SITEMAP_MODES,
   ENGINES,
   REGIONS,
+  REQUESTED_ENGINES,
   isStoredPageStub
 });

package/dist/cjs/utils/hast-fast.js

@@ -136,7 +136,7 @@ const pruneInPlace = (tree, keep) => {
     let write = 0;
     for (let read = 0; read < children.length; read++) {
       const child = children[read];
-      if (!keep(child)) {
+      if (!keep(child, parent)) {
         continue;
       }
       children[write++] = child;

package/dist/cjs/utils/parse-html.js

@@ -0,0 +1,113 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var parse_html_exports = {};
+__export(parse_html_exports, {
+  LARGE_DOCUMENT_CHARS: () => LARGE_DOCUMENT_CHARS,
+  MAX_ELEMENT_DEPTH: () => MAX_ELEMENT_DEPTH,
+  parseHtml: () => parseHtml,
+  stripScriptBodies: () => stripScriptBodies
+});
+module.exports = __toCommonJS(parse_html_exports);
+var import_hast_util_from_parse5 = require("hast-util-from-parse5");
+var import_parse5 = require("parse5");
+const MAX_ELEMENT_DEPTH = 256;
+const isParent = (node) => "childNodes" in node;
+const LINE_ELEMENTS = /* @__PURE__ */ new Set([
+  "p",
+  "div",
+  "li",
+  "tr",
+  "td",
+  "th",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "br",
+  "section",
+  "article",
+  "pre",
+  "blockquote"
+]);
+const textNode = (value, parentNode) => ({ nodeName: "#text", value, parentNode });
+const coalesceText = (parent) => {
+  const merged = [];
+  for (const child of parent.childNodes) {
+    const previous = merged.at(-1);
+    if (child.nodeName === "#text" && previous?.nodeName === "#text") {
+      previous.value += child.value;
+    } else {
+      merged.push(child);
+    }
+  }
+  parent.childNodes = merged;
+};
+const flattenDeep = (root, maxDepth) => {
+  const stack = [[root, 0]];
+  while (stack.length > 0) {
+    const [parent, depth] = stack.pop();
+    const children = parent.childNodes;
+    let unwrapped = false;
+    for (let index = 0; index < children.length; index++) {
+      const child = children[index];
+      if (!isParent(child)) {
+        continue;
+      }
+      if (depth + 1 > maxDepth && "tagName" in child) {
+        const replacement = [...child.childNodes];
+        if (LINE_ELEMENTS.has(child.tagName)) {
+          replacement.unshift(textNode("\n", parent));
+          replacement.push(textNode("\n", parent));
+        }
+        for (const node of replacement) {
+          node.parentNode = parent;
+        }
+        children.splice(index, 1, ...replacement);
+        index -= 1;
+        unwrapped = true;
+        continue;
+      }
+      stack.push([child, depth + 1]);
+    }
+    if (unwrapped) {
+      coalesceText(parent);
+    }
+  }
+};
+const SCRIPT_OR_STYLE = /<(script|style)\b([^>]*)>[\s\S]*?<\/\1\s*>/gi;
+const LARGE_DOCUMENT_CHARS = 2e6;
+const stripScriptBodies = (html) => html.replace(
+  SCRIPT_OR_STYLE,
+  (match, tag, attributes) => /application\/ld\+json/i.test(attributes) ? match : `<${tag}${attributes}></${tag}>`
+);
+const parseHtml = (html, options = {}) => {
+  const source = html.length > LARGE_DOCUMENT_CHARS ? stripScriptBodies(html) : html;
+  const document = options.fragment ? (0, import_parse5.parseFragment)(source, { scriptingEnabled: false }) : (0, import_parse5.parse)(source, { scriptingEnabled: false });
+  flattenDeep(document, MAX_ELEMENT_DEPTH);
+  return (0, import_hast_util_from_parse5.fromParse5)(document);
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  LARGE_DOCUMENT_CHARS,
+  MAX_ELEMENT_DEPTH,
+  parseHtml,
+  stripScriptBodies
+});

package/dist/cjs/utils/pre-text.js

@@ -0,0 +1,104 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var pre_text_exports = {};
+__export(pre_text_exports, {
+  preText: () => preText
+});
+module.exports = __toCommonJS(pre_text_exports);
+const LINE_BLOCK_TAGS = /* @__PURE__ */ new Set([
+  "address",
+  "article",
+  "aside",
+  "blockquote",
+  "dd",
+  "div",
+  "dl",
+  "dt",
+  "figcaption",
+  "figure",
+  "footer",
+  "form",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "header",
+  "hr",
+  "li",
+  "main",
+  "nav",
+  "ol",
+  "p",
+  "pre",
+  "section",
+  "table",
+  "tr",
+  "ul"
+]);
+const INLINE_CONTEXT_TAGS = /* @__PURE__ */ new Set(["span", "a", "b", "i", "em", "strong", "mark", "small", "u", "s", "label"]);
+const preText = (node) => {
+  let out = "";
+  const ensureLineStart = () => {
+    if (out.length > 0 && out.charCodeAt(out.length - 1) !== 10) {
+      out += "\n";
+    }
+  };
+  const visitChildren = (parent, inline) => {
+    let cells = 0;
+    for (const child of parent.children) {
+      const isCell = child.type === "element" && (child.tagName === "td" || child.tagName === "th");
+      visit(child, isCell ? cells++ : 0, inline);
+    }
+  };
+  const visitElement = (element, cellIndex, inline) => {
+    const tag = element.tagName;
+    if (tag === "br") {
+      out += "\n";
+      return;
+    }
+    if (cellIndex > 0) {
+      out += "	";
+    }
+    const block = !inline && LINE_BLOCK_TAGS.has(tag);
+    if (block) {
+      ensureLineStart();
+    }
+    visitChildren(element, inline || INLINE_CONTEXT_TAGS.has(tag));
+    if (block) {
+      ensureLineStart();
+    }
+  };
+  const visit = (current, cellIndex, inline) => {
+    if (current.type === "text") {
+      out += current.value;
+    } else if (current.type === "element") {
+      visitElement(current, cellIndex, inline);
+    } else if (current.type === "root") {
+      visitChildren(current, inline);
+    }
+  };
+  visit(node, 0, false);
+  return out;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  preText
+});

package/dist/detect-client-shell.js

@@ -0,0 +1,53 @@
+// src/detect-client-shell.ts
+var SHELL_SUSPICION_THRESHOLD = 300;
+var SHELL_EMPTY_THRESHOLD = 50;
+var BODY_PATTERN = /<body[^>]*>([\s\S]*?)<\/body>/i;
+var HEAD_PATTERN = /<head[^>]*>[\s\S]*?<\/head>/gi;
+var NOSCRIPT_PATTERN = /<noscript[^>]*>([\s\S]*?)<\/noscript>/gi;
+var INVISIBLE_ELEMENT_PATTERN = /<(script|style|template|svg|noscript)[^>]*>[\s\S]*?<\/\1>/gi;
+var COMMENT_PATTERN = /<!--[\s\S]*?-->/g;
+var TAG_PATTERN = /<[^>]+>/g;
+var MOUNT_POINT_PATTERN = /<[a-z][^>]*\sid=["'](?:root|app|__next|___gatsby|__nuxt|q-app|svelte|react-root)["']|<app-root[\s>]|\sdata-reactroot[\s>=]/i;
+var CHALLENGE_PATTERN = /cf-chl|challenge-platform|cf_chl_opt|just a moment|attention required|checking your browser|_incapsula_|px-captcha|datadome/i;
+var ENTITIES = {
+  "&nbsp;": " ",
+  "&amp;": "&",
+  "&lt;": "<",
+  "&gt;": ">",
+  "&quot;": '"',
+  "&#39;": "'"
+};
+var visibleTextOf = (markup) => markup.replace(COMMENT_PATTERN, " ").replace(INVISIBLE_ELEMENT_PATTERN, " ").replace(TAG_PATTERN, " ").replace(/&(?:nbsp|amp|lt|gt|quot|#39);/g, (entity) => ENTITIES[entity] ?? " ").replace(/\s+/g, " ").trim();
+var noscriptMentionsJavascript = (html) => {
+  for (const match of html.matchAll(NOSCRIPT_PATTERN)) {
+    if (/javascript/i.test(match[1] ?? "")) {
+      return true;
+    }
+  }
+  return false;
+};
+var detectClientShell = (html) => {
+  const body = BODY_PATTERN.exec(html)?.[1] ?? html.replace(HEAD_PATTERN, " ");
+  const visibleTextLength = visibleTextOf(body).length;
+  if (visibleTextLength >= SHELL_SUSPICION_THRESHOLD) {
+    return { isShell: false, visibleTextLength };
+  }
+  if (CHALLENGE_PATTERN.test(html)) {
+    return { isShell: true, reason: "anti-bot-challenge", visibleTextLength };
+  }
+  if (MOUNT_POINT_PATTERN.test(body)) {
+    return { isShell: true, reason: "spa-shell", visibleTextLength };
+  }
+  if (noscriptMentionsJavascript(html)) {
+    return { isShell: true, reason: "noscript-only", visibleTextLength };
+  }
+  if (visibleTextLength < SHELL_EMPTY_THRESHOLD) {
+    return { isShell: true, reason: "empty-body", visibleTextLength };
+  }
+  return { isShell: false, visibleTextLength };
+};
+export {
+  SHELL_EMPTY_THRESHOLD,
+  SHELL_SUSPICION_THRESHOLD,
+  detectClientShell
+};