webforai 3.0.0 → 4.0.0

package/dist/cjs/normalize/index.js

@@ -98,14 +98,16 @@ const katexDuplicates = (node) => {
   if (!(classes.includes("katex") || classes.includes("katex-display"))) {
     return [];
   }
-  const hasMathml = node.children.some((child) => (0, import_hast_fast.isElement)(child) && (0, import_hast_fast.classList)(child).includes("katex-mathml"));
+  const hasMathml = node.children.some(
+    (child) => (0, import_hast_fast.isElement)(child) && (0, import_hast_fast.classList)(child).includes("katex-mathml") && hasMathSource(child)
+  );
   if (!hasMathml) {
     return [];
   }
   return node.children.filter((child) => (0, import_hast_fast.isElement)(child) && (0, import_hast_fast.classList)(child).includes("katex-html"));
 };
 const mediawikiDuplicates = (node) => {
-  if (!(0, import_hast_fast.classList)(node).includes("mwe-math-element")) {
+  if (!(0, import_hast_fast.classList)(node).includes("mwe-math-element") || !hasMathSource(node)) {
     return [];
   }
   return node.children.filter(
@@ -116,8 +118,15 @@ const mathjaxDuplicates = (node) => {
   if (node.tagName !== "mjx-container") {
     return [];
   }
+  const hasMathml = node.children.some(
+    (child) => (0, import_hast_fast.isElement)(child) && child.tagName === "mjx-assistive-mml" && hasMathSource(child)
+  );
+  if (!hasMathml) {
+    return [];
+  }
   return node.children.filter((child) => (0, import_hast_fast.isElement)(child) && child.tagName !== "mjx-assistive-mml");
 };
+const hasMathSource = (node) => Boolean((0, import_hast_fast.findElement)(node, (element) => element.tagName === "math" && element.children.length > 0));
 const dedupeRenderedMath = (tree) => {
   const doomed = /* @__PURE__ */ new Set();
   (0, import_hast_fast.walk)(tree, (node) => {

package/dist/cjs/platform/client.js

@@ -50,14 +50,28 @@ const parseRetryAfter = (body, response) => {
   const header = Number(response.headers.get("retry-after"));
   return Number.isFinite(header) && header > 0 ? header : void 0;
 };
-const toApiError = (response, body) => {
+const dashboardHint = (status, dashboardUrl) => {
+  if (status === 401) {
+    return `create or check API keys at ${dashboardUrl}`;
+  }
+  if (status === 402) {
+    return `manage billing, credits and the spend cap at ${dashboardUrl}`;
+  }
+  return void 0;
+};
+const toApiError = (response, body, dashboardUrl) => {
   const envelope = body;
   const code = typeof envelope?.error?.code === "string" ? envelope.error.code : "invalid_response";
-  const message = typeof envelope?.error?.message === "string" ? envelope.error.message : `unexpected response (HTTP ${response.status})`;
+  let message = typeof envelope?.error?.message === "string" ? envelope.error.message : `unexpected response (HTTP ${response.status})`;
+  const hint = code === "invalid_response" ? void 0 : dashboardHint(response.status, dashboardUrl);
+  if (hint && !message.includes("/dashboard")) {
+    message = `${message} (${hint})`;
+  }
   return new import_error.PlatformApiError(code, message, response.status, parseRetryAfter(body, response));
 };
 const createPlatformClient = (options = {}) => {
   const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+  const dashboardUrl = `${baseUrl}/dashboard`;
   const fetchImpl = (0, import_fetch.resolveFetch)(options.fetch);
   const request = async (path, init) => {
     const headers = { accept: "application/json" };
@@ -68,7 +82,7 @@ const createPlatformClient = (options = {}) => {
       if (!options.apiKey) {
         throw new import_error.PlatformApiError(
           "missing_api_key",
-          "This endpoint needs an API key: pass `apiKey` to createPlatformClient (create one on the platform dashboard).",
+          `This endpoint needs an API key: pass \`apiKey\` to createPlatformClient (create one at ${dashboardUrl}; 1,000 free credits/month).`,
           0
         );
       }
@@ -77,18 +91,19 @@ const createPlatformClient = (options = {}) => {
     const response = await fetchImpl(`${baseUrl}${path}`, {
       method: init.method,
       headers,
-      body: init.body === void 0 ? void 0 : JSON.stringify(init.body)
+      body: init.body === void 0 ? void 0 : JSON.stringify(init.body),
+      ...init.signal ? { signal: init.signal } : {}
     });
     const body = await response.json().catch(() => void 0);
     if (!response.ok) {
-      throw toApiError(response, body);
+      throw toApiError(response, body, dashboardUrl);
     }
     if (body === void 0) {
       throw new import_error.PlatformApiError("invalid_response", "response body is not JSON", response.status);
     }
     return body;
   };
-  const getJob = (jobId) => request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: "GET", auth: true });
+  const getJob = (jobId, signal) => request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: "GET", auth: true, signal });
   const getJobResults = (jobId, opts) => {
     const query = opts?.cursor ? `?cursor=${encodeURIComponent(opts.cursor)}` : "";
     return request(`/v1/jobs/${encodeURIComponent(jobId)}/results${query}`, { method: "GET", auth: true });
@@ -104,6 +119,17 @@ const createPlatformClient = (options = {}) => {
     }
     return await response.json();
   };
+  const getJobOrBackOff = async (jobId, signal, interval) => {
+    try {
+      return await getJob(jobId, signal);
+    } catch (error) {
+      if (!(error instanceof import_error.PlatformApiError && error.status === 429)) {
+        throw error;
+      }
+      await sleep(error.retryAfter !== void 0 ? error.retryAfter * 1e3 : interval, signal);
+      return void 0;
+    }
+  };
   return {
     scrape: (opts) => request("/v1/scrape", { method: "POST", body: opts, auth: true }),
     scrapeAsync: (opts) => request("/v1/scrape", { method: "POST", body: { ...opts, async: true }, auth: true }),
@@ -114,17 +140,48 @@ const createPlatformClient = (options = {}) => {
     waitForJob: async (jobId, opts = {}) => {
       const interval = opts.pollIntervalMs ?? 2e3;
       const timeout = opts.timeoutMs ?? 10 * 60 * 1e3;
-      const startedAt = Date.now();
-      for (; ; ) {
-        const status = await getJob(jobId);
-        opts.onStatus?.(status);
-        if (TERMINAL_STATES.has(status.status)) {
-          return status;
-        }
-        if (Date.now() - startedAt + interval > timeout) {
-          throw new import_error.PlatformApiError("poll_timeout", `job ${jobId} still ${status.status} after ${timeout}ms`, 0);
+      if (!Number.isFinite(interval) || interval < 0 || interval > 2147483647) {
+        throw new RangeError("pollIntervalMs must be between 0 and 2147483647");
+      }
+      if (!Number.isFinite(timeout) || timeout <= 0 || timeout > 2147483647) {
+        throw new RangeError("timeoutMs must be greater than 0 and at most 2147483647");
+      }
+      opts.signal?.throwIfAborted();
+      const controller = new AbortController();
+      const { signal } = controller;
+      const onAbort = () => controller.abort(opts.signal?.reason);
+      opts.signal?.addEventListener("abort", onAbort, { once: true });
+      const timer = setTimeout(
+        () => controller.abort(new import_error.PlatformApiError("poll_timeout", `job ${jobId} did not finish within ${timeout}ms`, 0)),
+        timeout
+      );
+      let rejectOnAbort = () => {
+      };
+      const aborted = new Promise((_resolve, reject) => {
+        rejectOnAbort = () => reject(signal.reason);
+        signal.addEventListener("abort", rejectOnAbort, { once: true });
+      });
+      const poll = async () => {
+        for (; ; ) {
+          signal.throwIfAborted();
+          const status = await getJobOrBackOff(jobId, signal, interval);
+          signal.throwIfAborted();
+          if (status === void 0) {
+            continue;
+          }
+          opts.onStatus?.(status);
+          if (TERMINAL_STATES.has(status.status)) {
+            return status;
+          }
+          await sleep(interval, signal);
         }
-        await sleep(interval, opts.signal);
+      };
+      try {
+        return await Promise.race([poll(), aborted]);
+      } finally {
+        clearTimeout(timer);
+        opts.signal?.removeEventListener("abort", onAbort);
+        signal.removeEventListener("abort", rejectOnAbort);
       }
     },
     jobResults: async function* (jobId) {

package/dist/cjs/platform/index.js

@@ -18,10 +18,12 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var platform_exports = {};
 __export(platform_exports, {
+  CRAWL_SITEMAP_MODES: () => import_types.CRAWL_SITEMAP_MODES,
   DEFAULT_BASE_URL: () => import_client.DEFAULT_BASE_URL,
   ENGINES: () => import_types.ENGINES,
   PlatformApiError: () => import_error.PlatformApiError,
   REGIONS: () => import_types.REGIONS,
+  REQUESTED_ENGINES: () => import_types.REQUESTED_ENGINES,
   createPlatformClient: () => import_client.createPlatformClient,
   isStoredPageStub: () => import_types.isStoredPageStub
 });
@@ -31,10 +33,12 @@ var import_error = require("./error");
 var import_types = require("./types");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  CRAWL_SITEMAP_MODES,
   DEFAULT_BASE_URL,
   ENGINES,
   PlatformApiError,
   REGIONS,
+  REQUESTED_ENGINES,
   createPlatformClient,
   isStoredPageStub
 });

package/dist/cjs/platform/types.js

@@ -18,17 +18,23 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var types_exports = {};
 __export(types_exports, {
+  CRAWL_SITEMAP_MODES: () => CRAWL_SITEMAP_MODES,
   ENGINES: () => ENGINES,
   REGIONS: () => REGIONS,
+  REQUESTED_ENGINES: () => REQUESTED_ENGINES,
   isStoredPageStub: () => isStoredPageStub
 });
 module.exports = __toCommonJS(types_exports);
 const ENGINES = ["fetch", "browser", "proxy-fetch", "proxy-browser"];
-const REGIONS = ["auto", "us", "eu", "uk", "jp", "asia"];
+const REQUESTED_ENGINES = [...ENGINES, "auto"];
+const REGIONS = ["auto", "jp"];
+const CRAWL_SITEMAP_MODES = ["skip", "include", "only"];
 const isStoredPageStub = (item) => item.status === "ok" && "resultUrl" in item;
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  CRAWL_SITEMAP_MODES,
   ENGINES,
   REGIONS,
+  REQUESTED_ENGINES,
   isStoredPageStub
 });

package/dist/detect-client-shell.js

@@ -0,0 +1,53 @@
+// src/detect-client-shell.ts
+var SHELL_SUSPICION_THRESHOLD = 300;
+var SHELL_EMPTY_THRESHOLD = 50;
+var BODY_PATTERN = /<body[^>]*>([\s\S]*?)<\/body>/i;
+var HEAD_PATTERN = /<head[^>]*>[\s\S]*?<\/head>/gi;
+var NOSCRIPT_PATTERN = /<noscript[^>]*>([\s\S]*?)<\/noscript>/gi;
+var INVISIBLE_ELEMENT_PATTERN = /<(script|style|template|svg|noscript)[^>]*>[\s\S]*?<\/\1>/gi;
+var COMMENT_PATTERN = /<!--[\s\S]*?-->/g;
+var TAG_PATTERN = /<[^>]+>/g;
+var MOUNT_POINT_PATTERN = /<[a-z][^>]*\sid=["'](?:root|app|__next|___gatsby|__nuxt|q-app|svelte|react-root)["']|<app-root[\s>]|\sdata-reactroot[\s>=]/i;
+var CHALLENGE_PATTERN = /cf-chl|challenge-platform|cf_chl_opt|just a moment|attention required|checking your browser|_incapsula_|px-captcha|datadome/i;
+var ENTITIES = {
+  "&nbsp;": " ",
+  "&amp;": "&",
+  "&lt;": "<",
+  "&gt;": ">",
+  "&quot;": '"',
+  "&#39;": "'"
+};
+var visibleTextOf = (markup) => markup.replace(COMMENT_PATTERN, " ").replace(INVISIBLE_ELEMENT_PATTERN, " ").replace(TAG_PATTERN, " ").replace(/&(?:nbsp|amp|lt|gt|quot|#39);/g, (entity) => ENTITIES[entity] ?? " ").replace(/\s+/g, " ").trim();
+var noscriptMentionsJavascript = (html) => {
+  for (const match of html.matchAll(NOSCRIPT_PATTERN)) {
+    if (/javascript/i.test(match[1] ?? "")) {
+      return true;
+    }
+  }
+  return false;
+};
+var detectClientShell = (html) => {
+  const body = BODY_PATTERN.exec(html)?.[1] ?? html.replace(HEAD_PATTERN, " ");
+  const visibleTextLength = visibleTextOf(body).length;
+  if (visibleTextLength >= SHELL_SUSPICION_THRESHOLD) {
+    return { isShell: false, visibleTextLength };
+  }
+  if (CHALLENGE_PATTERN.test(html)) {
+    return { isShell: true, reason: "anti-bot-challenge", visibleTextLength };
+  }
+  if (MOUNT_POINT_PATTERN.test(body)) {
+    return { isShell: true, reason: "spa-shell", visibleTextLength };
+  }
+  if (noscriptMentionsJavascript(html)) {
+    return { isShell: true, reason: "noscript-only", visibleTextLength };
+  }
+  if (visibleTextLength < SHELL_EMPTY_THRESHOLD) {
+    return { isShell: true, reason: "empty-body", visibleTextLength };
+  }
+  return { isShell: false, visibleTextLength };
+};
+export {
+  SHELL_EMPTY_THRESHOLD,
+  SHELL_SUSPICION_THRESHOLD,
+  detectClientShell
+};

package/dist/extract-meta-refresh.js

@@ -0,0 +1,47 @@
+// src/extract-meta-refresh.ts
+var MAX_META_REFRESH_DELAY_SECONDS = 10;
+var META_TAG_PATTERN = /<meta\b[^>]*>/gi;
+var HTTP_EQUIV_REFRESH_PATTERN = /\bhttp-equiv\s*=\s*("\s*refresh\s*"|'\s*refresh\s*'|refresh\b)/i;
+var CONTENT_ATTR_PATTERN = /\bcontent\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/i;
+var CONTENT_VALUE_PATTERN = /^\s*(\d+(?:\.\d+)?)\s*[;,]\s*url\s*=\s*(?:"([^"]*)"|'([^']*)'|(.+?))\s*$/i;
+var resolveTarget = (raw, baseUrl) => {
+  try {
+    return baseUrl === void 0 ? new URL(raw) : new URL(raw, baseUrl);
+  } catch {
+    return void 0;
+  }
+};
+var extractMetaRefresh = (html, baseUrl) => {
+  for (const [tag] of html.matchAll(META_TAG_PATTERN)) {
+    if (!HTTP_EQUIV_REFRESH_PATTERN.test(tag)) {
+      continue;
+    }
+    const contentMatch = CONTENT_ATTR_PATTERN.exec(tag);
+    const content = contentMatch?.[1] ?? contentMatch?.[2] ?? contentMatch?.[3];
+    if (!content) {
+      continue;
+    }
+    const value = CONTENT_VALUE_PATTERN.exec(content);
+    if (!value) {
+      continue;
+    }
+    const delaySeconds = Number.parseFloat(value[1] ?? "");
+    const raw = (value[2] ?? value[3] ?? value[4] ?? "").trim();
+    if (!raw || delaySeconds > MAX_META_REFRESH_DELAY_SECONDS) {
+      continue;
+    }
+    const target = resolveTarget(raw, baseUrl);
+    if (!target || target.protocol !== "http:" && target.protocol !== "https:") {
+      continue;
+    }
+    if (baseUrl !== void 0 && resolveTarget(baseUrl, void 0)?.href === target.href) {
+      continue;
+    }
+    return { url: target.href, delaySeconds };
+  }
+  return void 0;
+};
+export {
+  MAX_META_REFRESH_DELAY_SECONDS,
+  extractMetaRefresh
+};

package/dist/extractors/lib/sanitize.js

@@ -374,7 +374,11 @@ var countAnchors = (element) => {
   }
   return count;
 };
+var LITERAL_TEXT_TAGS = /* @__PURE__ */ new Set(["code", "kbd", "samp", "var", "pre"]);
 var isMidArticleWidget = (element) => {
+  if (LITERAL_TEXT_TAGS.has(element.tagName)) {
+    return false;
+  }
   if (element.tagName === "form" || element.tagName === "fieldset") {
     return true;
   }

package/dist/html-to-mdast.js

@@ -12,13 +12,14 @@ import { definitionListHandler } from "./mdast-handlers/definition-list-handler.
 import { mathHandler } from "./mdast-handlers/math-handler.js";
 import { supSubHandler } from "./mdast-handlers/sup-sub-handler.js";
 import { normalizeHast } from "./normalize/index.js";
+import { cloneHast } from "./utils/hast-fast.js";
 import { getLangFromHast, getLangFromStr, getUrlFromHast } from "./utils/hast-utils.js";
 var htmlToMdast = (htmlOrHast, options) => {
   const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
   const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || getLangFromStr(htmlOrHast), fromHtml(htmlOrHast, { fragment: true })] : [defaultLang || getLangFromHast(htmlOrHast), htmlOrHast];
   const isOwned = typeof htmlOrHast === "string" || (options?.owned ?? false);
   const url = defaultUrl || getUrlFromHast(hast);
-  const extractedHast = pipeExtractors({ hast, lang, url, owned: isOwned }, extractors);
+  const extractedHast = pipeExtractors({ hast: isOwned ? hast : cloneHast(hast), lang, url, owned: true }, extractors);
   normalizeHast(extractedHast, options?.normalize);
   const mdast = toMdast(extractedHast, {
     handlers: {

package/dist/index.js

@@ -22,15 +22,29 @@ import {
   runAdapters,
   youtubeCaptionTracks
 } from "./adapters/index.js";
+import {
+  detectClientShell,
+  SHELL_EMPTY_THRESHOLD,
+  SHELL_SUSPICION_THRESHOLD
+} from "./detect-client-shell.js";
+import {
+  extractMetaRefresh,
+  MAX_META_REFRESH_DELAY_SECONDS
+} from "./extract-meta-refresh.js";
 import { extractMetadata, toFrontmatter } from "./metadata/index.js";
 import { normalizeHast } from "./normalize/index.js";
 export {
   BUILTIN_ADAPTERS,
+  MAX_META_REFRESH_DELAY_SECONDS,
+  SHELL_EMPTY_THRESHOLD,
+  SHELL_SUSPICION_THRESHOLD,
   adapterById,
   autoExtractor,
   buildPageSignature,
   createAutoExtractor,
   defineSelectorAdapter,
+  detectClientShell,
+  extractMetaRefresh,
   extractMetadata,
   htmlToMarkdown,
   htmlToMarkdownWithMetadata,

package/dist/link-replacer.js

@@ -1,17 +1,23 @@
 // src/link-replacer.ts
-var linkReplacer = (markdown, base) => {
-  const regex = /(!?\[.*?\]\()([^)\s]+)(\))/g;
-  return markdown.replace(regex, (match, pre, url, post) => {
-    if (/^(https?:|#)/.test(url)) {
-      return match;
-    }
-    try {
-      const absoluteUrl = new URL(url, base).href;
-      return `${pre}${absoluteUrl}${post}`;
-    } catch {
-      return match;
-    }
-  });
+var absoluteUrl = (url, base) => {
+  if (/^(?:[a-z][a-z\d+.-]*:|#)/i.test(url)) {
+    return url;
+  }
+  try {
+    return new URL(url, base).href;
+  } catch {
+    return url;
+  }
+};
+var linkReplacer = (node, base) => {
+  let result = node;
+  if (node.type === "link" || node.type === "image" || node.type === "definition") {
+    result = { ...node, url: absoluteUrl(node.url, base) };
+  }
+  if ("children" in result) {
+    return { ...result, children: result.children.map((child) => linkReplacer(child, base)) };
+  }
+  return result;
 };
 export {
   linkReplacer

package/dist/loaders/cf-puppeteer.js

@@ -1,14 +1,21 @@
 // src/loaders/cf-puppeteer.ts
-import puppeteer from "@cloudflare/puppeteer";
-var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+import puppeteer, { TimeoutError } from "@cloudflare/puppeteer";
 var loadHtml = async (url, ctx) => {
   const browser = await puppeteer.launch(ctx);
-  const page = await browser.newPage();
-  await page.goto(url);
-  const html = await page.content();
-  await Promise.race([page.waitForNetworkIdle(), sleep(1e4)]);
-  await page.close();
-  return html;
+  try {
+    const page = await browser.newPage();
+    await page.goto(url);
+    try {
+      await page.waitForNetworkIdle({ timeout: 1e4 });
+    } catch (error) {
+      if (!(error instanceof TimeoutError)) {
+        throw error;
+      }
+    }
+    return await page.content();
+  } finally {
+    await browser.close();
+  }
 };
 export {
   loadHtml

package/dist/loaders/fetch.js

@@ -1,8 +1,20 @@
 // src/loaders/fetch.ts
+import { extractMetaRefresh } from "../extract-meta-refresh.js";
 var USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
+var MAX_META_REFRESH_HOPS = 3;
 var loadHtml = async (url, userAgent = USER_AGENT) => {
-  const response = await fetch(url, { headers: { "User-Agent": userAgent } });
-  return response.text();
+  let current = url;
+  let html = "";
+  for (let hop = 0; hop <= MAX_META_REFRESH_HOPS; hop++) {
+    const response = await fetch(current, { headers: { "User-Agent": userAgent } });
+    html = await response.text();
+    const refresh = extractMetaRefresh(html, response.url || current);
+    if (!refresh) {
+      break;
+    }
+    current = refresh.url;
+  }
+  return html;
 };
 export {
   USER_AGENT,

package/dist/loaders/playwright.js

@@ -13,40 +13,42 @@ var SUPER_BYPASS_DEVICE = {
 var loadHtml = async (url, options) => {
   const { browser, waitUntil, timeout, superBypassMode } = options ?? {};
   const _browser = browser ?? await chromium.launch({ headless: true });
-  const context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : devices["Desktop Chrome"]);
-  if (superBypassMode) {
-    await context.addInitScript(() => {
-      Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
-      Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
-      Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
-    });
-  }
-  const page = await context.newPage();
-  if (superBypassMode) {
-    await page.route("**/*.js", (route) => {
-      if (route.request().url().includes("captcha-delivery")) {
-        return route.abort();
+  let context;
+  try {
+    context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : devices["Desktop Chrome"]);
+    if (superBypassMode) {
+      await context.addInitScript(() => {
+        Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
+        Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
+        Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
+      });
+    }
+    const page = await context.newPage();
+    if (superBypassMode) {
+      await page.route("**/*.js", (route) => {
+        if (route.request().url().includes("captcha-delivery")) {
+          return route.abort();
+        }
+        return route.continue();
+      });
+    }
+    await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
+    await page.evaluate(() => {
+      const elements = document.querySelectorAll("*");
+      for (const element of elements) {
+        const rect = element.getBoundingClientRect();
+        element.setAttribute("data-rwidth", rect.width.toString());
+        element.setAttribute("data-rheight", rect.height.toString());
       }
-      return route.continue();
     });
-  }
-  await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
-  await page.evaluate(() => {
-    const elements = document.querySelectorAll("*");
-    for (const element of elements) {
-      const rect = element.getBoundingClientRect();
-      element.setAttribute("data-rwidth", rect.width.toString());
-      element.setAttribute("data-rheight", rect.height.toString());
+    return await page.content();
+  } finally {
+    if (browser) {
+      await context?.close();
+    } else {
+      await _browser.close();
     }
-  });
-  const html = await page.content();
-  await page.close();
-  if (browser) {
-    await context.close();
-  } else {
-    await _browser.close();
   }
-  return html;
 };
 export {
   loadHtml

package/dist/loaders/puppeteer.js

@@ -1,6 +1,5 @@
 // src/loaders/puppeteer.ts
-import puppeteer from "puppeteer";
-var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+import puppeteer, { TimeoutError } from "puppeteer";
 var loadHtml = async (url, ctx) => {
   const browser = await puppeteer.launch(
     ctx || {
@@ -8,12 +7,20 @@ var loadHtml = async (url, ctx) => {
       args: ["--no-sandbox", "--disable-setuid-sandbox"]
     }
   );
-  const page = await browser.newPage();
-  await page.goto(url);
-  const html = await page.content();
-  await Promise.race([page.waitForNetworkIdle(), sleep(1e4)]);
-  await browser.close();
-  return html;
+  try {
+    const page = await browser.newPage();
+    await page.goto(url);
+    try {
+      await page.waitForNetworkIdle({ timeout: 1e4 });
+    } catch (error) {
+      if (!(error instanceof TimeoutError)) {
+        throw error;
+      }
+    }
+    return await page.content();
+  } finally {
+    await browser.close();
+  }
 };
 export {
   loadHtml

package/dist/md-splitter.js

@@ -3,15 +3,13 @@ import { mdastToMarkdown } from "./mdast-to-markdown.js";
 import { chunk } from "./utils/common.js";
 import { internalType, unwarpRoot, warpRoot } from "./utils/mdast-utils.js";
 var PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
-var getSplitterGenerator = function* () {
-  for (const splitter of PRIORITY_SPLITTERS) {
-    yield splitter;
-  }
-};
-var _mdastSplitter = async (contents, checker, splitterGenerator) => {
-  const splitter = splitterGenerator.next().value;
+var _mdastSplitter = async (contents, checker, priority, signal) => {
+  signal?.throwIfAborted();
+  const splitter = PRIORITY_SPLITTERS[priority];
   const markdown = mdastToMarkdown(warpRoot(contents));
-  if (await checker(markdown) || contents.length === 1) {
+  const accepted = await checker(markdown);
+  signal?.throwIfAborted();
+  if (accepted || contents.length === 1) {
     return [contents];
   }
   const chunked = splitter ? contents.reduce((acc, content) => {
@@ -22,11 +20,26 @@ var _mdastSplitter = async (contents, checker, splitterGenerator) => {
     acc[acc.length - 1].push(content);
     return acc;
   }, []) : chunk(contents, Math.ceil(contents.length / 2));
-  const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, splitterGenerator));
+  const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, priority + 1, signal));
   return Promise.all(splitting).then((chunks) => chunks.flat());
 };
-var mdastSplitter = (mdast, checker, _options) => {
-  return _mdastSplitter(unwarpRoot(mdast), checker, getSplitterGenerator());
+var mdastSplitter = async (mdast, checker, options) => {
+  const signal = options?.signal;
+  if (!signal) {
+    return _mdastSplitter(unwarpRoot(mdast), checker, 0);
+  }
+  signal.throwIfAborted();
+  let onAbort = () => {
+  };
+  const aborted = new Promise((_, reject) => {
+    onAbort = () => reject(signal.reason);
+    signal.addEventListener("abort", onAbort, { once: true });
+  });
+  try {
+    return await Promise.race([_mdastSplitter(unwarpRoot(mdast), checker, 0, signal), aborted]);
+  } finally {
+    signal.removeEventListener("abort", onAbort);
+  }
 };
 export {
   mdastSplitter