scavio 0.21.1 → 0.23.0

package/dist/index.cjs

@@ -1,7 +1,9 @@
 "use strict";
+var __create = Object.create;
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
 var __export = (target, all) => {
   for (var name in all)
@@ -15,6 +17,14 @@ var __copyProps = (to, from, except, desc) => {
   }
   return to;
 };
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
 // src/index.ts
@@ -30,6 +40,7 @@ __export(index_exports, {
   CostcoNamespace: () => CostcoNamespace,
   DEFAULT_MAX_REQUESTS_PER_SECOND: () => DEFAULT_MAX_REQUESTS_PER_SECOND,
   EbayNamespace: () => EbayNamespace,
+  ExtractNamespace: () => ExtractNamespace,
   G2Namespace: () => G2Namespace,
   GlassdoorNamespace: () => GlassdoorNamespace,
   GoogleAdsNamespace: () => GoogleAdsNamespace,
@@ -57,6 +68,7 @@ __export(index_exports, {
   ScavioAPIError: () => ScavioAPIError,
   ScavioConnectionError: () => ScavioConnectionError,
   ScavioError: () => ScavioError,
+  ScavioJobTimeoutError: () => ScavioJobTimeoutError,
   ScavioTimeoutError: () => ScavioTimeoutError,
   TargetNamespace: () => TargetNamespace,
   ThreadsNamespace: () => ThreadsNamespace,
@@ -68,7 +80,8 @@ __export(index_exports, {
   XNamespace: () => XNamespace,
   YelpNamespace: () => YelpNamespace,
   YouTubeNamespace: () => YouTubeNamespace,
-  ZillowNamespace: () => ZillowNamespace
+  ZillowNamespace: () => ZillowNamespace,
+  verifyWebhook: () => verifyWebhook
 });
 module.exports = __toCommonJS(index_exports);
 
@@ -155,6 +168,22 @@ var ScavioAPIError = class extends ScavioError {
     this.responseBody = responseBody;
   }
 };
+var ScavioJobTimeoutError = class extends ScavioError {
+  constructor(jobType, jobId, statusUrl, lastStatus) {
+    super(
+      `${jobType} ${jobId} was still ${lastStatus} when the wait timed out. It keeps running: poll extract.${jobType}Status("${jobId}") or cancel it.`
+    );
+    this.jobType = jobType;
+    this.jobId = jobId;
+    this.statusUrl = statusUrl;
+    this.lastStatus = lastStatus;
+    this.name = "ScavioJobTimeoutError";
+  }
+  jobType;
+  jobId;
+  statusUrl;
+  lastStatus;
+};
 
 // src/retry.ts
 var DEFAULT_RETRY_STATUSES = /* @__PURE__ */ new Set([
@@ -203,7 +232,7 @@ function parseRetryAfter(header) {
 var BASE_URL = "https://api.scavio.dev";
 var DEFAULT_TIMEOUT = 3e4;
 var DEFAULT_MAX_RETRIES = 2;
-var SDK_VERSION = true ? "0.21.1" : "0.0.0";
+var SDK_VERSION = true ? "0.23.0" : "0.0.0";
 var USER_AGENT = `scavio-js/${SDK_VERSION}`;
 function isServerRuntime() {
   const g = globalThis;
@@ -261,9 +290,18 @@ function getHeader(response, name) {
   }
   return null;
 }
+function buildQuery(query) {
+  if (!query) return "";
+  const params = new URLSearchParams();
+  for (const [key, value] of Object.entries(query)) {
+    if (value !== void 0) params.append(key, String(value));
+  }
+  const qs = params.toString();
+  return qs ? `?${qs}` : "";
+}
 async function request(options) {
-  const url = `${options.baseUrl}${options.path}`;
-  const headers = buildHeaders(options.apiKey);
+  const url = `${options.baseUrl}${options.path}${buildQuery(options.query)}`;
+  const headers = { ...buildHeaders(options.apiKey), ...options.headers };
   const retry = makeRetryConfig(
     options.maxRetries ?? DEFAULT_MAX_RETRIES
   );
@@ -493,6 +531,21 @@ var RedditNamespace = class {
   async trending() {
     return this.client._post("/api/v1/reddit/trending", {});
   }
+  /**
+   * Search Reddit comments by keyword. Each result carries the comment text,
+   * author, score and date, plus the post and subreddit it was left on.
+   * Costs 1 credit.
+   */
+  async searchComments(options) {
+    return this.client._post("/api/v1/reddit/search/comments", options);
+  }
+  /**
+   * Find subreddits by keyword, with subscriber counts, descriptions and NSFW
+   * flags. Costs 1 credit.
+   */
+  async searchSubreddits(options) {
+    return this.client._post("/api/v1/reddit/search/subreddits", options);
+  }
 };
 
 // src/namespaces/tiktok.ts
@@ -510,6 +563,14 @@ var TikTokNamespace = class {
   async video(options) {
     return this.client._post("/api/v1/tiktok/video", options);
   }
+  /**
+   * Spoken-word transcript of a TikTok video, as timestamped segments and
+   * full text. Pass `video_id` or `url`. Videos with no spoken captions
+   * (music, dance, muted clips) return `available: false`. 1 credit.
+   */
+  async videoTranscript(options) {
+    return this.client._post("/api/v1/tiktok/video/transcript", options);
+  }
   async videoComments(options) {
     return this.client._post("/api/v1/tiktok/video/comments", options);
   }
@@ -534,6 +595,107 @@ var TikTokNamespace = class {
   async userFollowings(options) {
     return this.client._post("/api/v1/tiktok/user/followings", options);
   }
+  /**
+   * Search terms trending on TikTok now (about 90 per call, US-weighted, no
+   * region filter). Costs 1 credit.
+   */
+  async trendingSearches() {
+    return this.client._post("/api/v1/tiktok/trending/searches", {});
+  }
+  /**
+   * A sample of videos trending on TikTok now, with author, stats, sound and
+   * hashtags. Each call returns about 6 videos whatever `count` is set to;
+   * call again for a fresh sample. No region filter. Costs 1 credit.
+   * Returns TikTok's raw payload: large numeric ids can lose precision as JSON
+   * numbers, so use the string forms (`id_str`, `sec_uid`).
+   */
+  async trendingVideos(options = {}) {
+    return this.client._post("/api/v1/tiktok/trending/videos", options);
+  }
+  /**
+   * TikTok search autocomplete suggestions for a keyword. Long-tail keywords
+   * can return none, which is still a billed answer. Costs 1 credit.
+   */
+  async searchSuggestions(options) {
+    return this.client._post("/api/v1/tiktok/search/suggestions", options);
+  }
+  /**
+   * A TikTok sound by id: title, artist, album, duration, how many videos use
+   * it, original vs commercial, play URL, cover, and chart rank when charting.
+   * Costs 1 credit.
+   * Returns TikTok's raw payload: large numeric ids can lose precision as JSON
+   * numbers, so use the string forms (`id_str`, `sec_uid`).
+   */
+  async music(options) {
+    return this.client._post("/api/v1/tiktok/music", options);
+  }
+  /**
+   * Videos that use a TikTok sound, about 14 per page (`count` is ignored
+   * upstream). Costs 1 credit.
+   * Returns TikTok's raw payload: large numeric ids can lose precision as JSON
+   * numbers, so use the string forms (`id_str`, `sec_uid`).
+   */
+  async musicVideos(options) {
+    return this.client._post("/api/v1/tiktok/music/videos", options);
+  }
+  /**
+   * Search TikTok sounds by keyword: id, title, artist, duration, usage count
+   * and play URL. Costs 1 credit.
+   * Returns TikTok's raw payload: large numeric ids can lose precision as JSON
+   * numbers, so use the string forms (`id_str`, `sec_uid`).
+   */
+  async searchMusic(options) {
+    return this.client._post("/api/v1/tiktok/search/music", options);
+  }
+  /**
+   * TikTok's Top 50 or Viral 50 sound chart, with each sound's details and
+   * trend. One global chart, no region filter. Costs 1 credit.
+   * Returns TikTok's raw payload: large numeric ids can lose precision as JSON
+   * numbers, so use the string forms (`id_str`, `sec_uid`).
+   */
+  async musicChart(options = {}) {
+    return this.client._post("/api/v1/tiktok/music/chart", options);
+  }
+  /**
+   * Creators similar to a TikTok user (about 30 per page), with follower,
+   * following, video and like counts, bio and verification. Costs 1 credit.
+   * Returns TikTok's raw payload: large numeric ids can lose precision as JSON
+   * numbers, so use the string forms (`id_str`, `sec_uid`).
+   */
+  async userSimilar(options) {
+    return this.client._post("/api/v1/tiktok/user/similar", options);
+  }
+  /**
+   * A TikTok user's playlists (series): id, name, video count and cover.
+   * Accounts without playlists return an empty list. Costs 1 credit.
+   */
+  async userPlaylists(options) {
+    return this.client._post("/api/v1/tiktok/user/playlists", options);
+  }
+  /**
+   * Videos in a TikTok playlist, with pagination. Costs 1 credit.
+   * Returns TikTok's raw payload: large numeric ids can lose precision as JSON
+   * numbers, so use the string forms (`id_str`, `sec_uid`).
+   */
+  async playlistVideos(options) {
+    return this.client._post("/api/v1/tiktok/playlist/videos", options);
+  }
+  /**
+   * Whether a TikTok user is live now: the user, live room status (2 = live,
+   * 4 = not live), title, start time and viewer counts. An unknown username is
+   * a billed 404. Costs 1 credit.
+   */
+  async userLive(options) {
+    return this.client._post("/api/v1/tiktok/user/live", options);
+  }
+  /**
+   * The region a TikTok account is registered in (ISO country code, not the
+   * creator's nationality), plus its user id and sec_user_id. An unknown
+   * username is a 404 and is not billed. Costs 1 credit.
+   */
+  async userRegion(options) {
+    return this.client._post("/api/v1/tiktok/user/region", options);
+  }
 };
 
 // src/namespaces/tiktok-shop.ts
@@ -826,6 +988,7 @@ var YouTubeNamespace = class {
   async commentReplies(options) {
     return this.client._post("/api/v1/youtube/comments/replies", options);
   }
+  /** Transcript / captions for a YouTube video. Costs 5 credits. */
   async transcript(options) {
     return this.client._post("/api/v1/youtube/transcript", options);
   }
@@ -857,6 +1020,21 @@ var YouTubeNamespace = class {
   async streams(options) {
     return this.client._post("/api/v1/youtube/streams", options);
   }
+  /**
+   * A single YouTube community post: text, author, likes, comment count,
+   * attached images, and a comments_cursor for communityPostComments.
+   * Costs 1 credit.
+   */
+  async communityPost(options) {
+    return this.client._post("/api/v1/youtube/community/post", options);
+  }
+  /**
+   * Comments on a YouTube community post, with likes, reply counts and
+   * pagination. Costs 1 credit.
+   */
+  async communityPostComments(options) {
+    return this.client._post("/api/v1/youtube/community/post/comments", options);
+  }
 };
 
 // src/namespaces/x.ts
@@ -898,6 +1076,27 @@ var XNamespace = class {
   async trending(options = {}) {
     return this.client._post("/api/v1/x/trending", options);
   }
+  /**
+   * Find X Communities by keyword, with member counts, topic and NSFW flag.
+   * Costs 1 credit.
+   */
+  async searchCommunities(options) {
+    return this.client._post("/api/v1/x/search/communities", options);
+  }
+  /**
+   * An X Community: name, description, creation date, member count, topic and
+   * rules. Costs 1 credit.
+   */
+  async community(options) {
+    return this.client._post("/api/v1/x/community", options);
+  }
+  /**
+   * Posts in an X Community, by relevance or recency, with pagination.
+   * Costs 1 credit.
+   */
+  async communityTweets(options) {
+    return this.client._post("/api/v1/x/community/tweets", options);
+  }
 };
 
 // src/namespaces/linkedin.ts
@@ -2792,10 +2991,128 @@ var MercadoLibreNamespace = class {
   }
 };
 
+// src/namespaces/extract.ts
+var TERMINAL = /* @__PURE__ */ new Set(["completed", "failed", "cancelled"]);
+var DEFAULT_POLL_INTERVAL = 2e3;
+var DEFAULT_WAIT_TIMEOUT = 6e5;
+var COLLECT_PAGE_SIZE = 100;
+function sleep2(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+var ExtractNamespace = class {
+  constructor(client) {
+    this.client = client;
+  }
+  client;
+  /**
+   * List a site's page URLs, gathered from its sitemaps and its start page,
+   * deduplicated, filtered by path and limited. Pages disallowed by the
+   * site's robots.txt are left out. 1 credit on the normal tier.
+   */
+  async map(options) {
+    return await this.client._post("/api/v1/extract/map", options);
+  }
+  async crawl(options, request2 = {}) {
+    return this.start("crawl", options, request2);
+  }
+  /** A crawl's progress and its page results, paged by cursor. Free. */
+  async crawlStatus(id, query) {
+    return this.status("crawl", id, query);
+  }
+  /**
+   * Stop a queued or running crawl. No further pages are fetched or billed;
+   * results so far stay available. A finished crawl answers 409. Free.
+   */
+  async cancelCrawl(id) {
+    return this.cancel("crawl", id);
+  }
+  async batch(options, request2 = {}) {
+    return this.start("batch", options, request2);
+  }
+  /** A batch's progress and its page results in input order, paged by cursor. Free. */
+  async batchStatus(id, query) {
+    return this.status("batch", id, query);
+  }
+  /** Stop a queued or running batch. A finished batch answers 409. Free. */
+  async cancelBatch(id) {
+    return this.cancel("batch", id);
+  }
+  async start(type, body, request2) {
+    const headers = request2.idempotencyKey ? { "Idempotency-Key": request2.idempotencyKey } : void 0;
+    const created = await this.client._post(
+      `/api/v1/extract/${type}`,
+      body,
+      headers
+    );
+    if (!request2.wait) return created;
+    return this.waitFor(type, created.data.id, created.data.status_url, request2);
+  }
+  async status(type, id, query = {}) {
+    return await this.client._get(`/api/v1/extract/${type}/${encodeURIComponent(id)}`, {
+      cursor: query.cursor,
+      limit: query.limit,
+      status: query.status,
+      include_content: query.include_content
+    });
+  }
+  async cancel(type, id) {
+    return await this.client._delete(
+      `/api/v1/extract/${type}/${encodeURIComponent(id)}`
+    );
+  }
+  async waitFor(type, id, statusUrl, request2) {
+    const interval = request2.pollInterval ?? DEFAULT_POLL_INTERVAL;
+    const deadline = Date.now() + (request2.waitTimeout ?? DEFAULT_WAIT_TIMEOUT);
+    for (; ; ) {
+      const probe = await this.status(type, id, { limit: 1, include_content: false });
+      if (TERMINAL.has(probe.data.status)) break;
+      if (Date.now() + interval > deadline) {
+        throw new ScavioJobTimeoutError(type, id, statusUrl, probe.data.status);
+      }
+      await sleep2(interval);
+    }
+    const results = [];
+    let cursor;
+    let last;
+    do {
+      last = await this.status(type, id, { cursor, limit: COLLECT_PAGE_SIZE });
+      results.push(...last.data.results);
+      cursor = last.data.next_cursor ?? void 0;
+    } while (cursor);
+    return { data: { ...last.data, results, next_cursor: null } };
+  }
+};
+
 // src/client.ts
 var DEFAULT_MAX_REQUESTS_PER_SECOND = 1;
 var MAX_REQUESTS_PER_SECOND = 50;
-var Scavio = class {
+var ScavioCore = class {
+  /**
+   * Read ANY web page and get it back as readability Markdown (the default),
+   * plain text, or raw HTML. Returns `{ url, format, mode, content,
+   * content_length }`.
+   *
+   * Extract is a core endpoint, not a platform: `scavio.extract({ url })`
+   * reads one page, and `scavio.extract.map()` / `.crawl()` / `.batch()` work
+   * on whole sites and URL lists.
+   *
+   * Credits are a function of `mode`, not a flat per-call constant:
+   * "normal" costs 1, "advanced" costs 1, "ultra" costs 2. Billing happens
+   * only on a successful extraction - a dead link, bot wall or timeout costs
+   * nothing.
+   *
+   * Start on "normal". Move to "advanced" when the page builds its content in
+   * the browser, and to "ultra" only when a bot wall blocks the other two.
+   *
+   * @example
+   * const page = await scavio.extract({ url: "https://example.com/pricing" });
+   * console.log(page.content);
+   */
+  async extract(options) {
+    return this._post("/api/v1/extract", options);
+  }
+};
+var Scavio = class extends ScavioCore {
   google;
   amazon;
   walmart;
@@ -2838,6 +3155,7 @@ var Scavio = class {
   maxRetries;
   rateLimiter;
   constructor(config) {
+    super();
     this.apiKey = config?.apiKey ?? process.env.SCAVIO_API_KEY ?? "";
     if (!this.apiKey) {
       throw new MissingAPIKeyError();
@@ -2888,9 +3206,19 @@ var Scavio = class {
     this.realtor = new RealtorNamespace(this);
     this.rightmove = new RightmoveNamespace(this);
     this.mercadolibre = new MercadoLibreNamespace(this);
+    const extractNs = new ExtractNamespace(this);
+    this.extract = Object.assign(ScavioCore.prototype.extract.bind(this), {
+      map: extractNs.map.bind(extractNs),
+      crawl: extractNs.crawl.bind(extractNs),
+      crawlStatus: extractNs.crawlStatus.bind(extractNs),
+      cancelCrawl: extractNs.cancelCrawl.bind(extractNs),
+      batch: extractNs.batch.bind(extractNs),
+      batchStatus: extractNs.batchStatus.bind(extractNs),
+      cancelBatch: extractNs.cancelBatch.bind(extractNs)
+    });
   }
   /** @internal */
-  async _post(path, body) {
+  async _post(path, body, headers) {
     return request({
       method: "POST",
       path,
@@ -2899,11 +3227,12 @@ var Scavio = class {
       timeout: this.timeout,
       maxRetries: this.maxRetries,
       rateLimiter: this.rateLimiter,
-      body
+      body,
+      headers
     });
   }
   /** @internal */
-  async _get(path) {
+  async _get(path, query) {
     return request({
       method: "GET",
       path,
@@ -2911,39 +3240,73 @@ var Scavio = class {
       baseUrl: this.baseUrl,
       timeout: this.timeout,
       maxRetries: this.maxRetries,
+      rateLimiter: this.rateLimiter,
+      query
+    });
+  }
+  /** @internal */
+  async _delete(path) {
+    return request({
+      method: "DELETE",
+      path,
+      apiKey: this.apiKey,
+      baseUrl: this.baseUrl,
+      timeout: this.timeout,
+      maxRetries: this.maxRetries,
       rateLimiter: this.rateLimiter
     });
   }
   async search(options) {
     return this.google.search(options);
   }
-  /**
-   * Read ANY web page and get it back as readability Markdown (the default),
-   * plain text, or raw HTML. Returns `{ url, format, mode, content,
-   * content_length }`.
-   *
-   * This is a core endpoint, not a platform, so it lives on the client itself:
-   * `scavio.extract({ url })`, never `scavio.extract.extract()`.
-   *
-   * Credits are a function of `mode`, not a flat per-call constant:
-   * "normal" costs 1, "advanced" costs 1, "ultra" costs 2. Billing happens
-   * only on a successful extraction - a dead link, bot wall or timeout costs
-   * nothing.
-   *
-   * Start on "normal". Move to "advanced" when the page builds its content in
-   * the browser, and to "ultra" only when a bot wall blocks the other two.
-   *
-   * @example
-   * const page = await scavio.extract({ url: "https://example.com/pricing" });
-   * console.log(page.content);
-   */
-  async extract(options) {
-    return this._post("/api/v1/extract", options);
-  }
   async getUsage() {
     return this._get("/api/v1/usage");
   }
 };
+
+// src/webhook.ts
+async function verifyWebhook(secret, header, body, toleranceSec = 300) {
+  if (!secret || !header) return false;
+  const parts = {};
+  for (const piece of header.split(",")) {
+    const i = piece.indexOf("=");
+    if (i > 0) parts[piece.slice(0, i).trim()] = piece.slice(i + 1).trim();
+  }
+  const t = Number(parts.t);
+  const v1 = parts.v1;
+  if (!parts.t || !Number.isFinite(t) || !v1 || !/^[0-9a-f]+$/i.test(v1)) return false;
+  if (Math.abs(Math.floor(Date.now() / 1e3) - t) > toleranceSec) return false;
+  const subtle = await getSubtle();
+  const enc = new TextEncoder();
+  const key = await subtle.importKey(
+    "raw",
+    enc.encode(secret),
+    { name: "HMAC", hash: "SHA-256" },
+    false,
+    ["sign"]
+  );
+  const bodyBytes = typeof body === "string" ? enc.encode(body) : body;
+  const prefix = enc.encode(`${t}.`);
+  const message = new Uint8Array(prefix.length + bodyBytes.length);
+  message.set(prefix, 0);
+  message.set(bodyBytes, prefix.length);
+  const mac = new Uint8Array(await subtle.sign("HMAC", key, message));
+  let expected = "";
+  for (const b of mac) expected += b.toString(16).padStart(2, "0");
+  return timingSafeEqualHex(expected, v1.toLowerCase());
+}
+function timingSafeEqualHex(a, b) {
+  if (a.length !== b.length) return false;
+  let diff = 0;
+  for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
+  return diff === 0;
+}
+async function getSubtle() {
+  const g = globalThis;
+  if (g.crypto?.subtle) return g.crypto.subtle;
+  const nodeCrypto = await import("crypto");
+  return nodeCrypto.webcrypto.subtle;
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   AirbnbNamespace,
@@ -2956,6 +3319,7 @@ var Scavio = class {
   CostcoNamespace,
   DEFAULT_MAX_REQUESTS_PER_SECOND,
   EbayNamespace,
+  ExtractNamespace,
   G2Namespace,
   GlassdoorNamespace,
   GoogleAdsNamespace,
@@ -2983,6 +3347,7 @@ var Scavio = class {
   ScavioAPIError,
   ScavioConnectionError,
   ScavioError,
+  ScavioJobTimeoutError,
   ScavioTimeoutError,
   TargetNamespace,
   ThreadsNamespace,
@@ -2994,6 +3359,7 @@ var Scavio = class {
   XNamespace,
   YelpNamespace,
   YouTubeNamespace,
-  ZillowNamespace
+  ZillowNamespace,
+  verifyWebhook
 });
 //# sourceMappingURL=index.cjs.map