webforai 2.1.0 → 3.0.0

package/dist/cjs/normalize/index.js

@@ -0,0 +1,168 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var normalize_exports = {};
+__export(normalize_exports, {
+  bestFromSrcset: () => bestFromSrcset,
+  dedupeRenderedMath: () => dedupeRenderedMath,
+  normalizeHast: () => normalizeHast,
+  promoteAriaHeadings: () => promoteAriaHeadings,
+  resolveImageSources: () => resolveImageSources
+});
+module.exports = __toCommonJS(normalize_exports);
+var import_hast_fast = require("../utils/hast-fast");
+const LAZY_SRC_ATTRIBUTES = [
+  "data-src",
+  "data-original",
+  "data-lazy-src",
+  "data-original-src",
+  "data-actualsrc",
+  "data-hi-res-src",
+  "data-echo"
+];
+const PLACEHOLDER_SRC = /^$|^data:image\/|^about:blank$|spacer\.gif$|blank\.(gif|png)$/i;
+const readSrcset = (element, attribute) => {
+  const raw = (0, import_hast_fast.property)(element, attribute);
+  if (typeof raw === "string") {
+    return raw;
+  }
+  if (Array.isArray(raw)) {
+    return raw.join(", ");
+  }
+  return void 0;
+};
+const bestFromSrcset = (srcset) => {
+  let best;
+  for (const entry of srcset.split(",")) {
+    const parts = entry.trim().split(/\s+/);
+    const url = parts[0];
+    if (!url) {
+      continue;
+    }
+    const descriptor = parts[1];
+    let weight = 1;
+    if (descriptor?.endsWith("w")) {
+      weight = Number.parseFloat(descriptor) || 1;
+    } else if (descriptor?.endsWith("x")) {
+      weight = (Number.parseFloat(descriptor) || 1) * 1e3;
+    }
+    if (!best || weight > best.weight) {
+      best = { url, weight };
+    }
+  }
+  return best?.url;
+};
+const resolveImageSources = (tree) => {
+  (0, import_hast_fast.walk)(tree, (node) => {
+    if (!(0, import_hast_fast.isElement)(node) || node.tagName !== "img") {
+      return;
+    }
+    const current = (0, import_hast_fast.stringProperty)(node, "src");
+    if (current && !PLACEHOLDER_SRC.test(current)) {
+      return;
+    }
+    for (const attribute of LAZY_SRC_ATTRIBUTES) {
+      const candidate = (0, import_hast_fast.stringProperty)(node, attribute);
+      if (candidate && !PLACEHOLDER_SRC.test(candidate)) {
+        node.properties.src = candidate;
+        return;
+      }
+    }
+    for (const attribute of ["srcSet", "srcset", "data-srcset"]) {
+      const srcset = readSrcset(node, attribute);
+      const best = srcset ? bestFromSrcset(srcset) : void 0;
+      if (best) {
+        node.properties.src = best;
+        return;
+      }
+    }
+  });
+};
+const katexDuplicates = (node) => {
+  const classes = (0, import_hast_fast.classList)(node);
+  if (!(classes.includes("katex") || classes.includes("katex-display"))) {
+    return [];
+  }
+  const hasMathml = node.children.some((child) => (0, import_hast_fast.isElement)(child) && (0, import_hast_fast.classList)(child).includes("katex-mathml"));
+  if (!hasMathml) {
+    return [];
+  }
+  return node.children.filter((child) => (0, import_hast_fast.isElement)(child) && (0, import_hast_fast.classList)(child).includes("katex-html"));
+};
+const mediawikiDuplicates = (node) => {
+  if (!(0, import_hast_fast.classList)(node).includes("mwe-math-element")) {
+    return [];
+  }
+  return node.children.filter(
+    (child) => (0, import_hast_fast.isElement)(child) && (0, import_hast_fast.classList)(child).some((name) => name.startsWith("mwe-math-fallback-image"))
+  );
+};
+const mathjaxDuplicates = (node) => {
+  if (node.tagName !== "mjx-container") {
+    return [];
+  }
+  return node.children.filter((child) => (0, import_hast_fast.isElement)(child) && child.tagName !== "mjx-assistive-mml");
+};
+const dedupeRenderedMath = (tree) => {
+  const doomed = /* @__PURE__ */ new Set();
+  (0, import_hast_fast.walk)(tree, (node) => {
+    if (!(0, import_hast_fast.isElement)(node)) {
+      return;
+    }
+    for (const duplicate of [...katexDuplicates(node), ...mathjaxDuplicates(node), ...mediawikiDuplicates(node)]) {
+      doomed.add(duplicate);
+    }
+  });
+  if (doomed.size > 0) {
+    (0, import_hast_fast.pruneInPlace)(tree, (node) => !((0, import_hast_fast.isElement)(node) && doomed.has(node)));
+  }
+};
+const promoteAriaHeadings = (tree) => {
+  (0, import_hast_fast.walk)(tree, (node) => {
+    if (!(0, import_hast_fast.isElement)(node) || (0, import_hast_fast.stringProperty)(node, "role") !== "heading") {
+      return;
+    }
+    if (/^h[1-6]$/.test(node.tagName)) {
+      return;
+    }
+    const level = (0, import_hast_fast.numericProperty)(node, "aria-level") ?? 2;
+    const clamped = Math.min(6, Math.max(1, Math.round(level)));
+    node.tagName = `h${clamped}`;
+  });
+};
+const normalizeHast = (tree, options = {}) => {
+  const { images = true, math = true, ariaHeadings = true } = options;
+  if (images) {
+    resolveImageSources(tree);
+  }
+  if (math) {
+    dedupeRenderedMath(tree);
+  }
+  if (ariaHeadings) {
+    promoteAriaHeadings(tree);
+  }
+  return tree;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  bestFromSrcset,
+  dedupeRenderedMath,
+  normalizeHast,
+  promoteAriaHeadings,
+  resolveImageSources
+});

package/dist/cjs/platform/client.js

@@ -0,0 +1,147 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var client_exports = {};
+__export(client_exports, {
+  DEFAULT_BASE_URL: () => DEFAULT_BASE_URL,
+  createPlatformClient: () => createPlatformClient
+});
+module.exports = __toCommonJS(client_exports);
+var import_error = require("./error");
+var import_fetch = require("./fetch");
+var import_types = require("./types");
+const DEFAULT_BASE_URL = "https://platform.webforai.dev";
+const TERMINAL_STATES = /* @__PURE__ */ new Set(["completed", "failed"]);
+const sleep = (ms, signal) => new Promise((resolve, reject) => {
+  if (signal?.aborted) {
+    reject(signal.reason instanceof Error ? signal.reason : new Error("aborted"));
+    return;
+  }
+  const timer = setTimeout(() => {
+    signal?.removeEventListener("abort", onAbort);
+    resolve();
+  }, ms);
+  const onAbort = () => {
+    clearTimeout(timer);
+    reject(signal?.reason instanceof Error ? signal.reason : new Error("aborted"));
+  };
+  signal?.addEventListener("abort", onAbort, { once: true });
+});
+const parseRetryAfter = (body, response) => {
+  const fromBody = body?.error?.retryAfter;
+  if (typeof fromBody === "number") {
+    return fromBody;
+  }
+  const header = Number(response.headers.get("retry-after"));
+  return Number.isFinite(header) && header > 0 ? header : void 0;
+};
+const toApiError = (response, body) => {
+  const envelope = body;
+  const code = typeof envelope?.error?.code === "string" ? envelope.error.code : "invalid_response";
+  const message = typeof envelope?.error?.message === "string" ? envelope.error.message : `unexpected response (HTTP ${response.status})`;
+  return new import_error.PlatformApiError(code, message, response.status, parseRetryAfter(body, response));
+};
+const createPlatformClient = (options = {}) => {
+  const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+  const fetchImpl = (0, import_fetch.resolveFetch)(options.fetch);
+  const request = async (path, init) => {
+    const headers = { accept: "application/json" };
+    if (init.body !== void 0) {
+      headers["content-type"] = "application/json";
+    }
+    if (init.auth) {
+      if (!options.apiKey) {
+        throw new import_error.PlatformApiError(
+          "missing_api_key",
+          "This endpoint needs an API key: pass `apiKey` to createPlatformClient (create one on the platform dashboard).",
+          0
+        );
+      }
+      headers.authorization = `Bearer ${options.apiKey}`;
+    }
+    const response = await fetchImpl(`${baseUrl}${path}`, {
+      method: init.method,
+      headers,
+      body: init.body === void 0 ? void 0 : JSON.stringify(init.body)
+    });
+    const body = await response.json().catch(() => void 0);
+    if (!response.ok) {
+      throw toApiError(response, body);
+    }
+    if (body === void 0) {
+      throw new import_error.PlatformApiError("invalid_response", "response body is not JSON", response.status);
+    }
+    return body;
+  };
+  const getJob = (jobId) => request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: "GET", auth: true });
+  const getJobResults = (jobId, opts) => {
+    const query = opts?.cursor ? `?cursor=${encodeURIComponent(opts.cursor)}` : "";
+    return request(`/v1/jobs/${encodeURIComponent(jobId)}/results${query}`, { method: "GET", auth: true });
+  };
+  const resolveStored = async (resultUrl) => {
+    const response = await fetchImpl(resultUrl, { method: "GET" });
+    if (!response.ok) {
+      throw new import_error.PlatformApiError(
+        "artifact_fetch_failed",
+        `stored result fetch failed (HTTP ${response.status}) \u2014 result URLs expire after ~24h`,
+        response.status
+      );
+    }
+    return await response.json();
+  };
+  return {
+    scrape: (opts) => request("/v1/scrape", { method: "POST", body: opts, auth: true }),
+    scrapeAsync: (opts) => request("/v1/scrape", { method: "POST", body: { ...opts, async: true }, auth: true }),
+    batch: (opts) => request("/v1/batch", { method: "POST", body: opts, auth: true }),
+    crawl: (opts) => request("/v1/crawl", { method: "POST", body: opts, auth: true }),
+    getJob,
+    getJobResults,
+    waitForJob: async (jobId, opts = {}) => {
+      const interval = opts.pollIntervalMs ?? 2e3;
+      const timeout = opts.timeoutMs ?? 10 * 60 * 1e3;
+      const startedAt = Date.now();
+      for (; ; ) {
+        const status = await getJob(jobId);
+        opts.onStatus?.(status);
+        if (TERMINAL_STATES.has(status.status)) {
+          return status;
+        }
+        if (Date.now() - startedAt + interval > timeout) {
+          throw new import_error.PlatformApiError("poll_timeout", `job ${jobId} still ${status.status} after ${timeout}ms`, 0);
+        }
+        await sleep(interval, opts.signal);
+      }
+    },
+    jobResults: async function* (jobId) {
+      let cursor;
+      do {
+        const page = await getJobResults(jobId, { cursor });
+        for (const item of page.results) {
+          yield (0, import_types.isStoredPageStub)(item) ? await resolveStored(item.resultUrl) : item;
+        }
+        cursor = page.cursor;
+      } while (cursor);
+    },
+    demoScrape: (opts) => request("/v1/demo/scrape", { method: "POST", body: opts, auth: false })
+  };
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  DEFAULT_BASE_URL,
+  createPlatformClient
+});

package/dist/cjs/platform/error.js

@@ -0,0 +1,36 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var error_exports = {};
+__export(error_exports, {
+  PlatformApiError: () => PlatformApiError
+});
+module.exports = __toCommonJS(error_exports);
+class PlatformApiError extends Error {
+  constructor(code, message, status, retryAfter) {
+    super(message);
+    this.name = "PlatformApiError";
+    this.code = code;
+    this.status = status;
+    this.retryAfter = retryAfter;
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  PlatformApiError
+});

package/dist/cjs/platform/fetch.js

@@ -0,0 +1,37 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var fetch_exports = {};
+__export(fetch_exports, {
+  resolveFetch: () => resolveFetch
+});
+module.exports = __toCommonJS(fetch_exports);
+const resolveFetch = (custom) => {
+  if (custom) {
+    return custom;
+  }
+  const globalFetch = globalThis.fetch;
+  if (typeof globalFetch !== "function") {
+    throw new Error("No global fetch in this runtime \u2014 pass a fetch implementation to createPlatformClient({ fetch })");
+  }
+  return globalFetch.bind(globalThis);
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  resolveFetch
+});

package/dist/cjs/platform/index.js

@@ -0,0 +1,40 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var platform_exports = {};
+__export(platform_exports, {
+  DEFAULT_BASE_URL: () => import_client.DEFAULT_BASE_URL,
+  ENGINES: () => import_types.ENGINES,
+  PlatformApiError: () => import_error.PlatformApiError,
+  REGIONS: () => import_types.REGIONS,
+  createPlatformClient: () => import_client.createPlatformClient,
+  isStoredPageStub: () => import_types.isStoredPageStub
+});
+module.exports = __toCommonJS(platform_exports);
+var import_client = require("./client");
+var import_error = require("./error");
+var import_types = require("./types");
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  DEFAULT_BASE_URL,
+  ENGINES,
+  PlatformApiError,
+  REGIONS,
+  createPlatformClient,
+  isStoredPageStub
+});

package/dist/cjs/platform/types.js

@@ -0,0 +1,34 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var types_exports = {};
+__export(types_exports, {
+  ENGINES: () => ENGINES,
+  REGIONS: () => REGIONS,
+  isStoredPageStub: () => isStoredPageStub
+});
+module.exports = __toCommonJS(types_exports);
+const ENGINES = ["fetch", "browser", "proxy-fetch", "proxy-browser"];
+const REGIONS = ["auto", "us", "eu", "uk", "jp", "asia"];
+const isStoredPageStub = (item) => item.status === "ok" && "resultUrl" in item;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  ENGINES,
+  REGIONS,
+  isStoredPageStub
+});