scavio 0.22.0 → 0.23.1

package/dist/index.js

@@ -81,6 +81,22 @@ var ScavioAPIError = class extends ScavioError {
     this.responseBody = responseBody;
   }
 };
+var ScavioJobTimeoutError = class extends ScavioError {
+  constructor(jobType, jobId, statusUrl, lastStatus) {
+    super(
+      `${jobType} ${jobId} was still ${lastStatus} when the wait timed out. It keeps running: poll extract.${jobType}Status("${jobId}") or cancel it.`
+    );
+    this.jobType = jobType;
+    this.jobId = jobId;
+    this.statusUrl = statusUrl;
+    this.lastStatus = lastStatus;
+    this.name = "ScavioJobTimeoutError";
+  }
+  jobType;
+  jobId;
+  statusUrl;
+  lastStatus;
+};
 
 // src/retry.ts
 var DEFAULT_RETRY_STATUSES = /* @__PURE__ */ new Set([
@@ -129,7 +145,7 @@ function parseRetryAfter(header) {
 var BASE_URL = "https://api.scavio.dev";
 var DEFAULT_TIMEOUT = 3e4;
 var DEFAULT_MAX_RETRIES = 2;
-var SDK_VERSION = true ? "0.22.0" : "0.0.0";
+var SDK_VERSION = true ? "0.23.1" : "0.0.0";
 var USER_AGENT = `scavio-js/${SDK_VERSION}`;
 function isServerRuntime() {
   const g = globalThis;
@@ -187,9 +203,18 @@ function getHeader(response, name) {
   }
   return null;
 }
+function buildQuery(query) {
+  if (!query) return "";
+  const params = new URLSearchParams();
+  for (const [key, value] of Object.entries(query)) {
+    if (value !== void 0) params.append(key, String(value));
+  }
+  const qs = params.toString();
+  return qs ? `?${qs}` : "";
+}
 async function request(options) {
-  const url = `${options.baseUrl}${options.path}`;
-  const headers = buildHeaders(options.apiKey);
+  const url = `${options.baseUrl}${options.path}${buildQuery(options.query)}`;
+  const headers = { ...buildHeaders(options.apiKey), ...options.headers };
   const retry = makeRetryConfig(
     options.maxRetries ?? DEFAULT_MAX_RETRIES
   );
@@ -451,6 +476,14 @@ var TikTokNamespace = class {
   async video(options) {
     return this.client._post("/api/v1/tiktok/video", options);
   }
+  /**
+   * Spoken-word transcript of a TikTok video, as timestamped segments and
+   * full text. Pass `video_id` or `url`. Videos with no spoken captions
+   * (music, dance, muted clips) return `available: false`. 1 credit.
+   */
+  async videoTranscript(options) {
+    return this.client._post("/api/v1/tiktok/video/transcript", options);
+  }
   async videoComments(options) {
     return this.client._post("/api/v1/tiktok/video/comments", options);
   }
@@ -2871,10 +2904,135 @@ var MercadoLibreNamespace = class {
   }
 };
 
+// src/namespaces/extract.ts
+var TERMINAL = /* @__PURE__ */ new Set(["completed", "failed", "cancelled"]);
+var DEFAULT_POLL_INTERVAL = 2e3;
+var DEFAULT_WAIT_TIMEOUT = 6e5;
+var COLLECT_PAGE_SIZE = 100;
+function sleep2(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function randomUUID() {
+  const g = globalThis;
+  if (typeof g.crypto?.randomUUID === "function") return g.crypto.randomUUID();
+  const nodeCrypto = await import("crypto");
+  return nodeCrypto.randomUUID();
+}
+var ExtractNamespace = class {
+  constructor(client) {
+    this.client = client;
+  }
+  client;
+  /**
+   * List a site's page URLs, gathered from its sitemaps and its start page,
+   * deduplicated, filtered by path and limited. Pages disallowed by the
+   * site's robots.txt are left out. 1 credit on the normal tier.
+   */
+  async map(options) {
+    return await this.client._post("/api/v1/extract/map", options);
+  }
+  async crawl(options, request2 = {}) {
+    return this.start("crawl", options, request2);
+  }
+  /** A crawl's progress and its page results, paged by cursor. Free. */
+  async crawlStatus(id, query) {
+    return this.status("crawl", id, query);
+  }
+  /**
+   * Stop a queued or running crawl. No further pages are fetched or billed;
+   * results so far stay available. A finished crawl answers 409. Free.
+   */
+  async cancelCrawl(id) {
+    return this.cancel("crawl", id);
+  }
+  async batch(options, request2 = {}) {
+    return this.start("batch", options, request2);
+  }
+  /** A batch's progress and its page results in input order, paged by cursor. Free. */
+  async batchStatus(id, query) {
+    return this.status("batch", id, query);
+  }
+  /** Stop a queued or running batch. A finished batch answers 409. Free. */
+  async cancelBatch(id) {
+    return this.cancel("batch", id);
+  }
+  async start(type, body, request2) {
+    const headers = { "Idempotency-Key": request2.idempotencyKey ?? await randomUUID() };
+    const created = await this.client._post(
+      `/api/v1/extract/${type}`,
+      body,
+      headers
+    );
+    if (!request2.wait) return created;
+    const done = await this.waitFor(type, created.data.id, created.data.status_url, request2);
+    return { ...done, created: created.data };
+  }
+  async status(type, id, query = {}) {
+    return await this.client._get(`/api/v1/extract/${type}/${encodeURIComponent(id)}`, {
+      cursor: query.cursor,
+      limit: query.limit,
+      status: query.status,
+      include_content: query.include_content
+    });
+  }
+  async cancel(type, id) {
+    return await this.client._delete(
+      `/api/v1/extract/${type}/${encodeURIComponent(id)}`
+    );
+  }
+  async waitFor(type, id, statusUrl, request2) {
+    const interval = request2.pollInterval ?? DEFAULT_POLL_INTERVAL;
+    const deadline = Date.now() + (request2.waitTimeout ?? DEFAULT_WAIT_TIMEOUT);
+    for (; ; ) {
+      const probe = await this.status(type, id, { limit: 1, include_content: false });
+      if (TERMINAL.has(probe.data.status)) break;
+      if (Date.now() + interval > deadline) {
+        throw new ScavioJobTimeoutError(type, id, statusUrl, probe.data.status);
+      }
+      await sleep2(interval);
+    }
+    const results = [];
+    let cursor;
+    let last;
+    do {
+      last = await this.status(type, id, { cursor, limit: COLLECT_PAGE_SIZE });
+      results.push(...last.data.results);
+      cursor = last.data.next_cursor ?? void 0;
+    } while (cursor);
+    return { data: { ...last.data, results, next_cursor: null } };
+  }
+};
+
 // src/client.ts
 var DEFAULT_MAX_REQUESTS_PER_SECOND = 1;
 var MAX_REQUESTS_PER_SECOND = 50;
-var Scavio = class {
+var ScavioCore = class {
+  /**
+   * Read ANY web page and get it back as readability Markdown (the default),
+   * plain text, or raw HTML. Returns `{ url, format, mode, content,
+   * content_length }`.
+   *
+   * Extract is a core endpoint, not a platform: `scavio.extract({ url })`
+   * reads one page, and `scavio.extract.map()` / `.crawl()` / `.batch()` work
+   * on whole sites and URL lists.
+   *
+   * Credits are a function of `mode`, not a flat per-call constant:
+   * "normal" costs 1, "advanced" costs 1, "ultra" costs 2. Billing happens
+   * only on a successful extraction - a dead link, bot wall or timeout costs
+   * nothing.
+   *
+   * Start on "normal". Move to "advanced" when the page builds its content in
+   * the browser, and to "ultra" only when a bot wall blocks the other two.
+   *
+   * @example
+   * const page = await scavio.extract({ url: "https://example.com/pricing" });
+   * console.log(page.content);
+   */
+  async extract(options) {
+    return this._post("/api/v1/extract", options);
+  }
+};
+var Scavio = class extends ScavioCore {
   google;
   amazon;
   walmart;
@@ -2917,6 +3075,7 @@ var Scavio = class {
   maxRetries;
   rateLimiter;
   constructor(config) {
+    super();
     this.apiKey = config?.apiKey ?? process.env.SCAVIO_API_KEY ?? "";
     if (!this.apiKey) {
       throw new MissingAPIKeyError();
@@ -2967,9 +3126,19 @@ var Scavio = class {
     this.realtor = new RealtorNamespace(this);
     this.rightmove = new RightmoveNamespace(this);
     this.mercadolibre = new MercadoLibreNamespace(this);
+    const extractNs = new ExtractNamespace(this);
+    this.extract = Object.assign(ScavioCore.prototype.extract.bind(this), {
+      map: extractNs.map.bind(extractNs),
+      crawl: extractNs.crawl.bind(extractNs),
+      crawlStatus: extractNs.crawlStatus.bind(extractNs),
+      cancelCrawl: extractNs.cancelCrawl.bind(extractNs),
+      batch: extractNs.batch.bind(extractNs),
+      batchStatus: extractNs.batchStatus.bind(extractNs),
+      cancelBatch: extractNs.cancelBatch.bind(extractNs)
+    });
   }
   /** @internal */
-  async _post(path, body) {
+  async _post(path, body, headers) {
     return request({
       method: "POST",
       path,
@@ -2978,11 +3147,12 @@ var Scavio = class {
       timeout: this.timeout,
       maxRetries: this.maxRetries,
       rateLimiter: this.rateLimiter,
-      body
+      body,
+      headers
     });
   }
   /** @internal */
-  async _get(path) {
+  async _get(path, query) {
     return request({
       method: "GET",
       path,
@@ -2990,39 +3160,73 @@ var Scavio = class {
       baseUrl: this.baseUrl,
       timeout: this.timeout,
       maxRetries: this.maxRetries,
+      rateLimiter: this.rateLimiter,
+      query
+    });
+  }
+  /** @internal */
+  async _delete(path) {
+    return request({
+      method: "DELETE",
+      path,
+      apiKey: this.apiKey,
+      baseUrl: this.baseUrl,
+      timeout: this.timeout,
+      maxRetries: this.maxRetries,
       rateLimiter: this.rateLimiter
     });
   }
   async search(options) {
     return this.google.search(options);
   }
-  /**
-   * Read ANY web page and get it back as readability Markdown (the default),
-   * plain text, or raw HTML. Returns `{ url, format, mode, content,
-   * content_length }`.
-   *
-   * This is a core endpoint, not a platform, so it lives on the client itself:
-   * `scavio.extract({ url })`, never `scavio.extract.extract()`.
-   *
-   * Credits are a function of `mode`, not a flat per-call constant:
-   * "normal" costs 1, "advanced" costs 1, "ultra" costs 2. Billing happens
-   * only on a successful extraction - a dead link, bot wall or timeout costs
-   * nothing.
-   *
-   * Start on "normal". Move to "advanced" when the page builds its content in
-   * the browser, and to "ultra" only when a bot wall blocks the other two.
-   *
-   * @example
-   * const page = await scavio.extract({ url: "https://example.com/pricing" });
-   * console.log(page.content);
-   */
-  async extract(options) {
-    return this._post("/api/v1/extract", options);
-  }
   async getUsage() {
     return this._get("/api/v1/usage");
   }
 };
+
+// src/webhook.ts
+async function verifyWebhook(secret, header, body, toleranceSec = 300) {
+  if (!secret || !header) return false;
+  const parts = {};
+  for (const piece of header.split(",")) {
+    const i = piece.indexOf("=");
+    if (i > 0) parts[piece.slice(0, i).trim()] = piece.slice(i + 1).trim();
+  }
+  const t = Number(parts.t);
+  const v1 = parts.v1;
+  if (!parts.t || !Number.isFinite(t) || !v1 || !/^[0-9a-f]+$/i.test(v1)) return false;
+  if (Math.abs(Math.floor(Date.now() / 1e3) - t) > toleranceSec) return false;
+  const subtle = await getSubtle();
+  const enc = new TextEncoder();
+  const key = await subtle.importKey(
+    "raw",
+    enc.encode(secret),
+    { name: "HMAC", hash: "SHA-256" },
+    false,
+    ["sign"]
+  );
+  const bodyBytes = typeof body === "string" ? enc.encode(body) : body;
+  const prefix = enc.encode(`${t}.`);
+  const message = new Uint8Array(prefix.length + bodyBytes.length);
+  message.set(prefix, 0);
+  message.set(bodyBytes, prefix.length);
+  const mac = new Uint8Array(await subtle.sign("HMAC", key, message));
+  let expected = "";
+  for (const b of mac) expected += b.toString(16).padStart(2, "0");
+  return timingSafeEqualHex(expected, v1.toLowerCase());
+}
+function timingSafeEqualHex(a, b) {
+  if (a.length !== b.length) return false;
+  let diff = 0;
+  for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
+  return diff === 0;
+}
+async function getSubtle() {
+  const g = globalThis;
+  if (g.crypto?.subtle) return g.crypto.subtle;
+  const nodeCrypto = await import("crypto");
+  return nodeCrypto.webcrypto.subtle;
+}
 export {
   AirbnbNamespace,
   AmazonNamespace,
@@ -3034,6 +3238,7 @@ export {
   CostcoNamespace,
   DEFAULT_MAX_REQUESTS_PER_SECOND,
   EbayNamespace,
+  ExtractNamespace,
   G2Namespace,
   GlassdoorNamespace,
   GoogleAdsNamespace,
@@ -3061,6 +3266,7 @@ export {
   ScavioAPIError,
   ScavioConnectionError,
   ScavioError,
+  ScavioJobTimeoutError,
   ScavioTimeoutError,
   TargetNamespace,
   ThreadsNamespace,
@@ -3072,6 +3278,7 @@ export {
   XNamespace,
   YelpNamespace,
   YouTubeNamespace,
-  ZillowNamespace
+  ZillowNamespace,
+  verifyWebhook
 };
 //# sourceMappingURL=index.js.map